lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <dlfcn.h>
   8
   9 #include <rte_jhash.h>
  10 #include <rte_hash_crc.h>
  11
  12 #include <rte_swx_port_ethdev.h>
  13 #include <rte_swx_port_fd.h>
  14 #include <rte_swx_port_ring.h>
  15 #include "rte_swx_port_source_sink.h"
  16
  17 #include <rte_swx_table_em.h>
  18 #include <rte_swx_table_wm.h>
  19
  20 #include "rte_swx_pipeline_internal.h"
  21
  22 #define CHECK(condition, err_code)                                             \
  23 do {                                                                           \
  24         if (!(condition))                                                      \
  25                 return -(err_code);                                            \
  26 } while (0)
  27
  28 #define CHECK_NAME(name, err_code)                                             \
  29         CHECK((name) &&                                                        \
  30               (name)[0] &&                                                     \
  31               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  32               err_code)
  33
  34 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  35         CHECK((instr) &&                                                       \
  36               (instr)[0] &&                                                    \
  37               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  38                RTE_SWX_INSTRUCTION_SIZE),                                      \
  39               err_code)
  40
  41 /*
  42  * Environment.
  43  */
  44 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  45
  46 #include <rte_malloc.h>
  47
  48 static void *
  49 env_malloc(size_t size, size_t alignment, int numa_node)
  50 {
  51         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  52 }
  53
  54 static void
  55 env_free(void *start, size_t size __rte_unused)
  56 {
  57         rte_free(start);
  58 }
  59
  60 #else
  61
  62 #include <numa.h>
  63
  64 static void *
  65 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  66 {
  67         void *start;
  68
  69         if (numa_available() == -1)
  70                 return NULL;
  71
  72         start = numa_alloc_onnode(size, numa_node);
  73         if (!start)
  74                 return NULL;
  75
  76         memset(start, 0, size);
  77         return start;
  78 }
  79
  80 static void
  81 env_free(void *start, size_t size)
  82 {
  83         if (numa_available() == -1)
  84                 return;
  85
  86         numa_free(start, size);
  87 }
  88
  89 #endif
  90
  91 /*
  92  * Struct.
  93  */
  94 static struct struct_type *
  95 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  96 {
  97         struct struct_type *elem;
  98
  99         TAILQ_FOREACH(elem, &p->struct_types, node)
 100                 if (strcmp(elem->name, name) == 0)
 101                         return elem;
 102
 103         return NULL;
 104 }
 105
 106 static struct field *
 107 struct_type_field_find(struct struct_type *st, const char *name)
 108 {
 109         uint32_t i;
 110
 111         for (i = 0; i < st->n_fields; i++) {
 112                 struct field *f = &st->fields[i];
 113
 114                 if (strcmp(f->name, name) == 0)
 115                         return f;
 116         }
 117
 118         return NULL;
 119 }
 120
 121 int
 122 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 123                                       const char *name,
 124                                       struct rte_swx_field_params *fields,
 125                                       uint32_t n_fields,
 126                                       int last_field_has_variable_size)
 127 {
 128         struct struct_type *st;
 129         uint32_t i;
 130
 131         CHECK(p, EINVAL);
 132         CHECK_NAME(name, EINVAL);
 133         CHECK(fields, EINVAL);
 134         CHECK(n_fields, EINVAL);
 135
 136         for (i = 0; i < n_fields; i++) {
 137                 struct rte_swx_field_params *f = &fields[i];
 138                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 139                 uint32_t j;
 140
 141                 CHECK_NAME(f->name, EINVAL);
 142                 CHECK(f->n_bits, EINVAL);
 143                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 144                 CHECK((f->n_bits & 7) == 0, EINVAL);
 145
 146                 for (j = 0; j < i; j++) {
 147                         struct rte_swx_field_params *f_prev = &fields[j];
 148
 149                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 150                 }
 151         }
 152
 153         CHECK(!struct_type_find(p, name), EEXIST);
 154
 155         /* Node allocation. */
 156         st = calloc(1, sizeof(struct struct_type));
 157         CHECK(st, ENOMEM);
 158
 159         st->fields = calloc(n_fields, sizeof(struct field));
 160         if (!st->fields) {
 161                 free(st);
 162                 CHECK(0, ENOMEM);
 163         }
 164
 165         /* Node initialization. */
 166         strcpy(st->name, name);
 167         for (i = 0; i < n_fields; i++) {
 168                 struct field *dst = &st->fields[i];
 169                 struct rte_swx_field_params *src = &fields[i];
 170                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 171
 172                 strcpy(dst->name, src->name);
 173                 dst->n_bits = src->n_bits;
 174                 dst->offset = st->n_bits;
 175                 dst->var_size = var_size;
 176
 177                 st->n_bits += src->n_bits;
 178                 st->n_bits_min += var_size ? 0 : src->n_bits;
 179         }
 180         st->n_fields = n_fields;
 181         st->var_size = last_field_has_variable_size;
 182
 183         /* Node add to tailq. */
 184         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 185
 186         return 0;
 187 }
 188
 189 static int
 190 struct_build(struct rte_swx_pipeline *p)
 191 {
 192         uint32_t i;
 193
 194         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 195                 struct thread *t = &p->threads[i];
 196
 197                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 198                 CHECK(t->structs, ENOMEM);
 199         }
 200
 201         return 0;
 202 }
 203
 204 static void
 205 struct_build_free(struct rte_swx_pipeline *p)
 206 {
 207         uint32_t i;
 208
 209         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 210                 struct thread *t = &p->threads[i];
 211
 212                 free(t->structs);
 213                 t->structs = NULL;
 214         }
 215 }
 216
 217 static void
 218 struct_free(struct rte_swx_pipeline *p)
 219 {
 220         struct_build_free(p);
 221
 222         /* Struct types. */
 223         for ( ; ; ) {
 224                 struct struct_type *elem;
 225
 226                 elem = TAILQ_FIRST(&p->struct_types);
 227                 if (!elem)
 228                         break;
 229
 230                 TAILQ_REMOVE(&p->struct_types, elem, node);
 231                 free(elem->fields);
 232                 free(elem);
 233         }
 234 }
 235
 236 /*
 237  * Input port.
 238  */
 239 static struct port_in_type *
 240 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 241 {
 242         struct port_in_type *elem;
 243
 244         if (!name)
 245                 return NULL;
 246
 247         TAILQ_FOREACH(elem, &p->port_in_types, node)
 248                 if (strcmp(elem->name, name) == 0)
 249                         return elem;
 250
 251         return NULL;
 252 }
 253
 254 int
 255 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 256                                        const char *name,
 257                                        struct rte_swx_port_in_ops *ops)
 258 {
 259         struct port_in_type *elem;
 260
 261         CHECK(p, EINVAL);
 262         CHECK_NAME(name, EINVAL);
 263         CHECK(ops, EINVAL);
 264         CHECK(ops->create, EINVAL);
 265         CHECK(ops->free, EINVAL);
 266         CHECK(ops->pkt_rx, EINVAL);
 267         CHECK(ops->stats_read, EINVAL);
 268
 269         CHECK(!port_in_type_find(p, name), EEXIST);
 270
 271         /* Node allocation. */
 272         elem = calloc(1, sizeof(struct port_in_type));
 273         CHECK(elem, ENOMEM);
 274
 275         /* Node initialization. */
 276         strcpy(elem->name, name);
 277         memcpy(&elem->ops, ops, sizeof(*ops));
 278
 279         /* Node add to tailq. */
 280         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 281
 282         return 0;
 283 }
 284
 285 static struct port_in *
 286 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 287 {
 288         struct port_in *port;
 289
 290         TAILQ_FOREACH(port, &p->ports_in, node)
 291                 if (port->id == port_id)
 292                         return port;
 293
 294         return NULL;
 295 }
 296
 297 int
 298 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 299                                 uint32_t port_id,
 300                                 const char *port_type_name,
 301                                 void *args)
 302 {
 303         struct port_in_type *type = NULL;
 304         struct port_in *port = NULL;
 305         void *obj = NULL;
 306
 307         CHECK(p, EINVAL);
 308
 309         CHECK(!port_in_find(p, port_id), EINVAL);
 310
 311         CHECK_NAME(port_type_name, EINVAL);
 312         type = port_in_type_find(p, port_type_name);
 313         CHECK(type, EINVAL);
 314
 315         obj = type->ops.create(args);
 316         CHECK(obj, ENODEV);
 317
 318         /* Node allocation. */
 319         port = calloc(1, sizeof(struct port_in));
 320         CHECK(port, ENOMEM);
 321
 322         /* Node initialization. */
 323         port->type = type;
 324         port->obj = obj;
 325         port->id = port_id;
 326
 327         /* Node add to tailq. */
 328         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 329         if (p->n_ports_in < port_id + 1)
 330                 p->n_ports_in = port_id + 1;
 331
 332         return 0;
 333 }
 334
 335 static int
 336 port_in_build(struct rte_swx_pipeline *p)
 337 {
 338         struct port_in *port;
 339         uint32_t i;
 340
 341         CHECK(p->n_ports_in, EINVAL);
 342         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 343
 344         for (i = 0; i < p->n_ports_in; i++)
 345                 CHECK(port_in_find(p, i), EINVAL);
 346
 347         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 348         CHECK(p->in, ENOMEM);
 349
 350         TAILQ_FOREACH(port, &p->ports_in, node) {
 351                 struct port_in_runtime *in = &p->in[port->id];
 352
 353                 in->pkt_rx = port->type->ops.pkt_rx;
 354                 in->obj = port->obj;
 355         }
 356
 357         return 0;
 358 }
 359
 360 static void
 361 port_in_build_free(struct rte_swx_pipeline *p)
 362 {
 363         free(p->in);
 364         p->in = NULL;
 365 }
 366
 367 static void
 368 port_in_free(struct rte_swx_pipeline *p)
 369 {
 370         port_in_build_free(p);
 371
 372         /* Input ports. */
 373         for ( ; ; ) {
 374                 struct port_in *port;
 375
 376                 port = TAILQ_FIRST(&p->ports_in);
 377                 if (!port)
 378                         break;
 379
 380                 TAILQ_REMOVE(&p->ports_in, port, node);
 381                 port->type->ops.free(port->obj);
 382                 free(port);
 383         }
 384
 385         /* Input port types. */
 386         for ( ; ; ) {
 387                 struct port_in_type *elem;
 388
 389                 elem = TAILQ_FIRST(&p->port_in_types);
 390                 if (!elem)
 391                         break;
 392
 393                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 394                 free(elem);
 395         }
 396 }
 397
 398 /*
 399  * Output port.
 400  */
 401 static struct port_out_type *
 402 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 403 {
 404         struct port_out_type *elem;
 405
 406         if (!name)
 407                 return NULL;
 408
 409         TAILQ_FOREACH(elem, &p->port_out_types, node)
 410                 if (!strcmp(elem->name, name))
 411                         return elem;
 412
 413         return NULL;
 414 }
 415
 416 int
 417 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 418                                         const char *name,
 419                                         struct rte_swx_port_out_ops *ops)
 420 {
 421         struct port_out_type *elem;
 422
 423         CHECK(p, EINVAL);
 424         CHECK_NAME(name, EINVAL);
 425         CHECK(ops, EINVAL);
 426         CHECK(ops->create, EINVAL);
 427         CHECK(ops->free, EINVAL);
 428         CHECK(ops->pkt_tx, EINVAL);
 429         CHECK(ops->pkt_fast_clone_tx, EINVAL);
 430         CHECK(ops->pkt_clone_tx, EINVAL);
 431         CHECK(ops->stats_read, EINVAL);
 432
 433         CHECK(!port_out_type_find(p, name), EEXIST);
 434
 435         /* Node allocation. */
 436         elem = calloc(1, sizeof(struct port_out_type));
 437         CHECK(elem, ENOMEM);
 438
 439         /* Node initialization. */
 440         strcpy(elem->name, name);
 441         memcpy(&elem->ops, ops, sizeof(*ops));
 442
 443         /* Node add to tailq. */
 444         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 445
 446         return 0;
 447 }
 448
 449 static struct port_out *
 450 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 451 {
 452         struct port_out *port;
 453
 454         TAILQ_FOREACH(port, &p->ports_out, node)
 455                 if (port->id == port_id)
 456                         return port;
 457
 458         return NULL;
 459 }
 460
 461 int
 462 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 463                                  uint32_t port_id,
 464                                  const char *port_type_name,
 465                                  void *args)
 466 {
 467         struct port_out_type *type = NULL;
 468         struct port_out *port = NULL;
 469         void *obj = NULL;
 470
 471         CHECK(p, EINVAL);
 472
 473         CHECK(!port_out_find(p, port_id), EINVAL);
 474
 475         CHECK_NAME(port_type_name, EINVAL);
 476         type = port_out_type_find(p, port_type_name);
 477         CHECK(type, EINVAL);
 478
 479         obj = type->ops.create(args);
 480         CHECK(obj, ENODEV);
 481
 482         /* Node allocation. */
 483         port = calloc(1, sizeof(struct port_out));
 484         CHECK(port, ENOMEM);
 485
 486         /* Node initialization. */
 487         port->type = type;
 488         port->obj = obj;
 489         port->id = port_id;
 490
 491         /* Node add to tailq. */
 492         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 493         if (p->n_ports_out < port_id + 1)
 494                 p->n_ports_out = port_id + 1;
 495
 496         return 0;
 497 }
 498
 499 static int
 500 port_out_build(struct rte_swx_pipeline *p)
 501 {
 502         struct port_out *port;
 503         uint32_t i;
 504
 505         CHECK(p->n_ports_out, EINVAL);
 506
 507         for (i = 0; i < p->n_ports_out; i++)
 508                 CHECK(port_out_find(p, i), EINVAL);
 509
 510         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 511         CHECK(p->out, ENOMEM);
 512
 513         TAILQ_FOREACH(port, &p->ports_out, node) {
 514                 struct port_out_runtime *out = &p->out[port->id];
 515
 516                 out->pkt_tx = port->type->ops.pkt_tx;
 517                 out->pkt_fast_clone_tx = port->type->ops.pkt_fast_clone_tx;
 518                 out->pkt_clone_tx = port->type->ops.pkt_clone_tx;
 519                 out->flush = port->type->ops.flush;
 520                 out->obj = port->obj;
 521         }
 522
 523         return 0;
 524 }
 525
 526 static void
 527 port_out_build_free(struct rte_swx_pipeline *p)
 528 {
 529         free(p->out);
 530         p->out = NULL;
 531 }
 532
 533 static void
 534 port_out_free(struct rte_swx_pipeline *p)
 535 {
 536         port_out_build_free(p);
 537
 538         /* Output ports. */
 539         for ( ; ; ) {
 540                 struct port_out *port;
 541
 542                 port = TAILQ_FIRST(&p->ports_out);
 543                 if (!port)
 544                         break;
 545
 546                 TAILQ_REMOVE(&p->ports_out, port, node);
 547                 port->type->ops.free(port->obj);
 548                 free(port);
 549         }
 550
 551         /* Output port types. */
 552         for ( ; ; ) {
 553                 struct port_out_type *elem;
 554
 555                 elem = TAILQ_FIRST(&p->port_out_types);
 556                 if (!elem)
 557                         break;
 558
 559                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 560                 free(elem);
 561         }
 562 }
 563
 564 /*
 565  * Packet mirroring.
 566  */
 567 int
 568 rte_swx_pipeline_mirroring_config(struct rte_swx_pipeline *p,
 569                                   struct rte_swx_pipeline_mirroring_params *params)
 570 {
 571         CHECK(p, EINVAL);
 572         CHECK(params, EINVAL);
 573         CHECK(params->n_slots, EINVAL);
 574         CHECK(params->n_sessions, EINVAL);
 575         CHECK(!p->build_done, EEXIST);
 576
 577         p->n_mirroring_slots = rte_align32pow2(params->n_slots);
 578         if (p->n_mirroring_slots > 64)
 579                 p->n_mirroring_slots = 64;
 580
 581         p->n_mirroring_sessions = rte_align32pow2(params->n_sessions);
 582
 583         return 0;
 584 }
 585
 586 static void
 587 mirroring_build_free(struct rte_swx_pipeline *p)
 588 {
 589         uint32_t i;
 590
 591         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 592                 struct thread *t = &p->threads[i];
 593
 594                 /* mirroring_slots. */
 595                 free(t->mirroring_slots);
 596                 t->mirroring_slots = NULL;
 597         }
 598
 599         /* mirroring_sessions. */
 600         free(p->mirroring_sessions);
 601         p->mirroring_sessions = NULL;
 602 }
 603
 604 static void
 605 mirroring_free(struct rte_swx_pipeline *p)
 606 {
 607         mirroring_build_free(p);
 608 }
 609
 610 static int
 611 mirroring_build(struct rte_swx_pipeline *p)
 612 {
 613         uint32_t i;
 614
 615         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 616                 struct thread *t = &p->threads[i];
 617
 618                 /* mirroring_slots. */
 619                 t->mirroring_slots = calloc(p->n_mirroring_slots, sizeof(uint32_t));
 620                 if (!t->mirroring_slots)
 621                         goto error;
 622         }
 623
 624         /* mirroring_sessions. */
 625         p->mirroring_sessions = calloc(p->n_mirroring_sessions, sizeof(struct mirroring_session));
 626         if (!p->mirroring_sessions)
 627                 goto error;
 628
 629         return 0;
 630
 631 error:
 632         mirroring_build_free(p);
 633         return -ENOMEM;
 634 }
 635
 636 /*
 637  * Extern object.
 638  */
 639 static struct extern_type *
 640 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 641 {
 642         struct extern_type *elem;
 643
 644         TAILQ_FOREACH(elem, &p->extern_types, node)
 645                 if (strcmp(elem->name, name) == 0)
 646                         return elem;
 647
 648         return NULL;
 649 }
 650
 651 static struct extern_type_member_func *
 652 extern_type_member_func_find(struct extern_type *type, const char *name)
 653 {
 654         struct extern_type_member_func *elem;
 655
 656         TAILQ_FOREACH(elem, &type->funcs, node)
 657                 if (strcmp(elem->name, name) == 0)
 658                         return elem;
 659
 660         return NULL;
 661 }
 662
 663 static struct extern_obj *
 664 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 665 {
 666         struct extern_obj *elem;
 667
 668         TAILQ_FOREACH(elem, &p->extern_objs, node)
 669                 if (strcmp(elem->name, name) == 0)
 670                         return elem;
 671
 672         return NULL;
 673 }
 674
 675 static struct extern_type_member_func *
 676 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 677                              const char *name,
 678                              struct extern_obj **obj)
 679 {
 680         struct extern_obj *object;
 681         struct extern_type_member_func *func;
 682         char *object_name, *func_name;
 683
 684         if (name[0] != 'e' || name[1] != '.')
 685                 return NULL;
 686
 687         object_name = strdup(&name[2]);
 688         if (!object_name)
 689                 return NULL;
 690
 691         func_name = strchr(object_name, '.');
 692         if (!func_name) {
 693                 free(object_name);
 694                 return NULL;
 695         }
 696
 697         *func_name = 0;
 698         func_name++;
 699
 700         object = extern_obj_find(p, object_name);
 701         if (!object) {
 702                 free(object_name);
 703                 return NULL;
 704         }
 705
 706         func = extern_type_member_func_find(object->type, func_name);
 707         if (!func) {
 708                 free(object_name);
 709                 return NULL;
 710         }
 711
 712         if (obj)
 713                 *obj = object;
 714
 715         free(object_name);
 716         return func;
 717 }
 718
 719 static struct field *
 720 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 721                                const char *name,
 722                                struct extern_obj **object)
 723 {
 724         struct extern_obj *obj;
 725         struct field *f;
 726         char *obj_name, *field_name;
 727
 728         if ((name[0] != 'e') || (name[1] != '.'))
 729                 return NULL;
 730
 731         obj_name = strdup(&name[2]);
 732         if (!obj_name)
 733                 return NULL;
 734
 735         field_name = strchr(obj_name, '.');
 736         if (!field_name) {
 737                 free(obj_name);
 738                 return NULL;
 739         }
 740
 741         *field_name = 0;
 742         field_name++;
 743
 744         obj = extern_obj_find(p, obj_name);
 745         if (!obj) {
 746                 free(obj_name);
 747                 return NULL;
 748         }
 749
 750         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 751         if (!f) {
 752                 free(obj_name);
 753                 return NULL;
 754         }
 755
 756         if (object)
 757                 *object = obj;
 758
 759         free(obj_name);
 760         return f;
 761 }
 762
 763 int
 764 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 765         const char *name,
 766         const char *mailbox_struct_type_name,
 767         rte_swx_extern_type_constructor_t constructor,
 768         rte_swx_extern_type_destructor_t destructor)
 769 {
 770         struct extern_type *elem;
 771         struct struct_type *mailbox_struct_type;
 772
 773         CHECK(p, EINVAL);
 774
 775         CHECK_NAME(name, EINVAL);
 776         CHECK(!extern_type_find(p, name), EEXIST);
 777
 778         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 779         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 780         CHECK(mailbox_struct_type, EINVAL);
 781         CHECK(!mailbox_struct_type->var_size, EINVAL);
 782
 783         CHECK(constructor, EINVAL);
 784         CHECK(destructor, EINVAL);
 785
 786         /* Node allocation. */
 787         elem = calloc(1, sizeof(struct extern_type));
 788         CHECK(elem, ENOMEM);
 789
 790         /* Node initialization. */
 791         strcpy(elem->name, name);
 792         elem->mailbox_struct_type = mailbox_struct_type;
 793         elem->constructor = constructor;
 794         elem->destructor = destructor;
 795         TAILQ_INIT(&elem->funcs);
 796
 797         /* Node add to tailq. */
 798         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 799
 800         return 0;
 801 }
 802
 803 int
 804 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 805         const char *extern_type_name,
 806         const char *name,
 807         rte_swx_extern_type_member_func_t member_func)
 808 {
 809         struct extern_type *type;
 810         struct extern_type_member_func *type_member;
 811
 812         CHECK(p, EINVAL);
 813
 814         CHECK_NAME(extern_type_name, EINVAL);
 815         type = extern_type_find(p, extern_type_name);
 816         CHECK(type, EINVAL);
 817         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 818
 819         CHECK_NAME(name, EINVAL);
 820         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 821
 822         CHECK(member_func, EINVAL);
 823
 824         /* Node allocation. */
 825         type_member = calloc(1, sizeof(struct extern_type_member_func));
 826         CHECK(type_member, ENOMEM);
 827
 828         /* Node initialization. */
 829         strcpy(type_member->name, name);
 830         type_member->func = member_func;
 831         type_member->id = type->n_funcs;
 832
 833         /* Node add to tailq. */
 834         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 835         type->n_funcs++;
 836
 837         return 0;
 838 }
 839
 840 int
 841 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 842                                       const char *extern_type_name,
 843                                       const char *name,
 844                                       const char *args)
 845 {
 846         struct extern_type *type;
 847         struct extern_obj *obj;
 848         void *obj_handle;
 849
 850         CHECK(p, EINVAL);
 851
 852         CHECK_NAME(extern_type_name, EINVAL);
 853         type = extern_type_find(p, extern_type_name);
 854         CHECK(type, EINVAL);
 855
 856         CHECK_NAME(name, EINVAL);
 857         CHECK(!extern_obj_find(p, name), EEXIST);
 858
 859         /* Node allocation. */
 860         obj = calloc(1, sizeof(struct extern_obj));
 861         CHECK(obj, ENOMEM);
 862
 863         /* Object construction. */
 864         obj_handle = type->constructor(args);
 865         if (!obj_handle) {
 866                 free(obj);
 867                 CHECK(0, ENODEV);
 868         }
 869
 870         /* Node initialization. */
 871         strcpy(obj->name, name);
 872         obj->type = type;
 873         obj->obj = obj_handle;
 874         obj->struct_id = p->n_structs;
 875         obj->id = p->n_extern_objs;
 876
 877         /* Node add to tailq. */
 878         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 879         p->n_extern_objs++;
 880         p->n_structs++;
 881
 882         return 0;
 883 }
 884
 885 static int
 886 extern_obj_build(struct rte_swx_pipeline *p)
 887 {
 888         uint32_t i;
 889
 890         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 891                 struct thread *t = &p->threads[i];
 892                 struct extern_obj *obj;
 893
 894                 t->extern_objs = calloc(p->n_extern_objs,
 895                                         sizeof(struct extern_obj_runtime));
 896                 CHECK(t->extern_objs, ENOMEM);
 897
 898                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 899                         struct extern_obj_runtime *r =
 900                                 &t->extern_objs[obj->id];
 901                         struct extern_type_member_func *func;
 902                         uint32_t mailbox_size =
 903                                 obj->type->mailbox_struct_type->n_bits / 8;
 904
 905                         r->obj = obj->obj;
 906
 907                         r->mailbox = calloc(1, mailbox_size);
 908                         CHECK(r->mailbox, ENOMEM);
 909
 910                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 911                                 r->funcs[func->id] = func->func;
 912
 913                         t->structs[obj->struct_id] = r->mailbox;
 914                 }
 915         }
 916
 917         return 0;
 918 }
 919
 920 static void
 921 extern_obj_build_free(struct rte_swx_pipeline *p)
 922 {
 923         uint32_t i;
 924
 925         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 926                 struct thread *t = &p->threads[i];
 927                 uint32_t j;
 928
 929                 if (!t->extern_objs)
 930                         continue;
 931
 932                 for (j = 0; j < p->n_extern_objs; j++) {
 933                         struct extern_obj_runtime *r = &t->extern_objs[j];
 934
 935                         free(r->mailbox);
 936                 }
 937
 938                 free(t->extern_objs);
 939                 t->extern_objs = NULL;
 940         }
 941 }
 942
 943 static void
 944 extern_obj_free(struct rte_swx_pipeline *p)
 945 {
 946         extern_obj_build_free(p);
 947
 948         /* Extern objects. */
 949         for ( ; ; ) {
 950                 struct extern_obj *elem;
 951
 952                 elem = TAILQ_FIRST(&p->extern_objs);
 953                 if (!elem)
 954                         break;
 955
 956                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 957                 if (elem->obj)
 958                         elem->type->destructor(elem->obj);
 959                 free(elem);
 960         }
 961
 962         /* Extern types. */
 963         for ( ; ; ) {
 964                 struct extern_type *elem;
 965
 966                 elem = TAILQ_FIRST(&p->extern_types);
 967                 if (!elem)
 968                         break;
 969
 970                 TAILQ_REMOVE(&p->extern_types, elem, node);
 971
 972                 for ( ; ; ) {
 973                         struct extern_type_member_func *func;
 974
 975                         func = TAILQ_FIRST(&elem->funcs);
 976                         if (!func)
 977                                 break;
 978
 979                         TAILQ_REMOVE(&elem->funcs, func, node);
 980                         free(func);
 981                 }
 982
 983                 free(elem);
 984         }
 985 }
 986
 987 /*
 988  * Extern function.
 989  */
 990 static struct extern_func *
 991 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 992 {
 993         struct extern_func *elem;
 994
 995         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 996                 if (strcmp(elem->name, name) == 0)
 997                         return elem;
 998
 999         return NULL;
1000 }
1001
1002 static struct extern_func *
1003 extern_func_parse(struct rte_swx_pipeline *p,
1004                   const char *name)
1005 {
1006         if (name[0] != 'f' || name[1] != '.')
1007                 return NULL;
1008
1009         return extern_func_find(p, &name[2]);
1010 }
1011
1012 static struct field *
1013 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1014                                 const char *name,
1015                                 struct extern_func **function)
1016 {
1017         struct extern_func *func;
1018         struct field *f;
1019         char *func_name, *field_name;
1020
1021         if ((name[0] != 'f') || (name[1] != '.'))
1022                 return NULL;
1023
1024         func_name = strdup(&name[2]);
1025         if (!func_name)
1026                 return NULL;
1027
1028         field_name = strchr(func_name, '.');
1029         if (!field_name) {
1030                 free(func_name);
1031                 return NULL;
1032         }
1033
1034         *field_name = 0;
1035         field_name++;
1036
1037         func = extern_func_find(p, func_name);
1038         if (!func) {
1039                 free(func_name);
1040                 return NULL;
1041         }
1042
1043         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1044         if (!f) {
1045                 free(func_name);
1046                 return NULL;
1047         }
1048
1049         if (function)
1050                 *function = func;
1051
1052         free(func_name);
1053         return f;
1054 }
1055
1056 int
1057 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1058                                       const char *name,
1059                                       const char *mailbox_struct_type_name,
1060                                       rte_swx_extern_func_t func)
1061 {
1062         struct extern_func *f;
1063         struct struct_type *mailbox_struct_type;
1064
1065         CHECK(p, EINVAL);
1066
1067         CHECK_NAME(name, EINVAL);
1068         CHECK(!extern_func_find(p, name), EEXIST);
1069
1070         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1071         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1072         CHECK(mailbox_struct_type, EINVAL);
1073         CHECK(!mailbox_struct_type->var_size, EINVAL);
1074
1075         CHECK(func, EINVAL);
1076
1077         /* Node allocation. */
1078         f = calloc(1, sizeof(struct extern_func));
1079         CHECK(func, ENOMEM);
1080
1081         /* Node initialization. */
1082         strcpy(f->name, name);
1083         f->mailbox_struct_type = mailbox_struct_type;
1084         f->func = func;
1085         f->struct_id = p->n_structs;
1086         f->id = p->n_extern_funcs;
1087
1088         /* Node add to tailq. */
1089         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1090         p->n_extern_funcs++;
1091         p->n_structs++;
1092
1093         return 0;
1094 }
1095
1096 static int
1097 extern_func_build(struct rte_swx_pipeline *p)
1098 {
1099         uint32_t i;
1100
1101         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1102                 struct thread *t = &p->threads[i];
1103                 struct extern_func *func;
1104
1105                 /* Memory allocation. */
1106                 t->extern_funcs = calloc(p->n_extern_funcs,
1107                                          sizeof(struct extern_func_runtime));
1108                 CHECK(t->extern_funcs, ENOMEM);
1109
1110                 /* Extern function. */
1111                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1112                         struct extern_func_runtime *r =
1113                                 &t->extern_funcs[func->id];
1114                         uint32_t mailbox_size =
1115                                 func->mailbox_struct_type->n_bits / 8;
1116
1117                         r->func = func->func;
1118
1119                         r->mailbox = calloc(1, mailbox_size);
1120                         CHECK(r->mailbox, ENOMEM);
1121
1122                         t->structs[func->struct_id] = r->mailbox;
1123                 }
1124         }
1125
1126         return 0;
1127 }
1128
1129 static void
1130 extern_func_build_free(struct rte_swx_pipeline *p)
1131 {
1132         uint32_t i;
1133
1134         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1135                 struct thread *t = &p->threads[i];
1136                 uint32_t j;
1137
1138                 if (!t->extern_funcs)
1139                         continue;
1140
1141                 for (j = 0; j < p->n_extern_funcs; j++) {
1142                         struct extern_func_runtime *r = &t->extern_funcs[j];
1143
1144                         free(r->mailbox);
1145                 }
1146
1147                 free(t->extern_funcs);
1148                 t->extern_funcs = NULL;
1149         }
1150 }
1151
1152 static void
1153 extern_func_free(struct rte_swx_pipeline *p)
1154 {
1155         extern_func_build_free(p);
1156
1157         for ( ; ; ) {
1158                 struct extern_func *elem;
1159
1160                 elem = TAILQ_FIRST(&p->extern_funcs);
1161                 if (!elem)
1162                         break;
1163
1164                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1165                 free(elem);
1166         }
1167 }
1168
1169 /*
1170  * Hash function.
1171  */
1172 static struct hash_func *
1173 hash_func_find(struct rte_swx_pipeline *p, const char *name)
1174 {
1175         struct hash_func *elem;
1176
1177         TAILQ_FOREACH(elem, &p->hash_funcs, node)
1178                 if (strcmp(elem->name, name) == 0)
1179                         return elem;
1180
1181         return NULL;
1182 }
1183
1184 int
1185 rte_swx_pipeline_hash_func_register(struct rte_swx_pipeline *p,
1186                                     const char *name,
1187                                     rte_swx_hash_func_t func)
1188 {
1189         struct hash_func *f;
1190
1191         CHECK(p, EINVAL);
1192
1193         CHECK_NAME(name, EINVAL);
1194         CHECK(!hash_func_find(p, name), EEXIST);
1195
1196         CHECK(func, EINVAL);
1197
1198         /* Node allocation. */
1199         f = calloc(1, sizeof(struct hash_func));
1200         CHECK(func, ENOMEM);
1201
1202         /* Node initialization. */
1203         strcpy(f->name, name);
1204         f->func = func;
1205         f->id = p->n_hash_funcs;
1206
1207         /* Node add to tailq. */
1208         TAILQ_INSERT_TAIL(&p->hash_funcs, f, node);
1209         p->n_hash_funcs++;
1210
1211         return 0;
1212 }
1213
1214 static int
1215 hash_func_build(struct rte_swx_pipeline *p)
1216 {
1217         struct hash_func *func;
1218
1219         /* Memory allocation. */
1220         p->hash_func_runtime = calloc(p->n_hash_funcs, sizeof(struct hash_func_runtime));
1221         CHECK(p->hash_func_runtime, ENOMEM);
1222
1223         /* Hash function. */
1224         TAILQ_FOREACH(func, &p->hash_funcs, node) {
1225                 struct hash_func_runtime *r = &p->hash_func_runtime[func->id];
1226
1227                 r->func = func->func;
1228         }
1229
1230         return 0;
1231 }
1232
1233 static void
1234 hash_func_build_free(struct rte_swx_pipeline *p)
1235 {
1236         free(p->hash_func_runtime);
1237         p->hash_func_runtime = NULL;
1238 }
1239
1240 static void
1241 hash_func_free(struct rte_swx_pipeline *p)
1242 {
1243         hash_func_build_free(p);
1244
1245         for ( ; ; ) {
1246                 struct hash_func *elem;
1247
1248                 elem = TAILQ_FIRST(&p->hash_funcs);
1249                 if (!elem)
1250                         break;
1251
1252                 TAILQ_REMOVE(&p->hash_funcs, elem, node);
1253                 free(elem);
1254         }
1255 }
1256
1257 /*
1258  * Header.
1259  */
1260 static struct header *
1261 header_find(struct rte_swx_pipeline *p, const char *name)
1262 {
1263         struct header *elem;
1264
1265         TAILQ_FOREACH(elem, &p->headers, node)
1266                 if (strcmp(elem->name, name) == 0)
1267                         return elem;
1268
1269         return NULL;
1270 }
1271
1272 static struct header *
1273 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1274 {
1275         struct header *elem;
1276
1277         TAILQ_FOREACH(elem, &p->headers, node)
1278                 if (elem->struct_id == struct_id)
1279                         return elem;
1280
1281         return NULL;
1282 }
1283
1284 static struct header *
1285 header_parse(struct rte_swx_pipeline *p,
1286              const char *name)
1287 {
1288         if (name[0] != 'h' || name[1] != '.')
1289                 return NULL;
1290
1291         return header_find(p, &name[2]);
1292 }
1293
1294 static struct field *
1295 header_field_parse(struct rte_swx_pipeline *p,
1296                    const char *name,
1297                    struct header **header)
1298 {
1299         struct header *h;
1300         struct field *f;
1301         char *header_name, *field_name;
1302
1303         if ((name[0] != 'h') || (name[1] != '.'))
1304                 return NULL;
1305
1306         header_name = strdup(&name[2]);
1307         if (!header_name)
1308                 return NULL;
1309
1310         field_name = strchr(header_name, '.');
1311         if (!field_name) {
1312                 free(header_name);
1313                 return NULL;
1314         }
1315
1316         *field_name = 0;
1317         field_name++;
1318
1319         h = header_find(p, header_name);
1320         if (!h) {
1321                 free(header_name);
1322                 return NULL;
1323         }
1324
1325         f = struct_type_field_find(h->st, field_name);
1326         if (!f) {
1327                 free(header_name);
1328                 return NULL;
1329         }
1330
1331         if (header)
1332                 *header = h;
1333
1334         free(header_name);
1335         return f;
1336 }
1337
1338 int
1339 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1340                                         const char *name,
1341                                         const char *struct_type_name)
1342 {
1343         struct struct_type *st;
1344         struct header *h;
1345         size_t n_headers_max;
1346
1347         CHECK(p, EINVAL);
1348         CHECK_NAME(name, EINVAL);
1349         CHECK_NAME(struct_type_name, EINVAL);
1350
1351         CHECK(!header_find(p, name), EEXIST);
1352
1353         st = struct_type_find(p, struct_type_name);
1354         CHECK(st, EINVAL);
1355
1356         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1357         CHECK(p->n_headers < n_headers_max, ENOSPC);
1358
1359         /* Node allocation. */
1360         h = calloc(1, sizeof(struct header));
1361         CHECK(h, ENOMEM);
1362
1363         /* Node initialization. */
1364         strcpy(h->name, name);
1365         h->st = st;
1366         h->struct_id = p->n_structs;
1367         h->id = p->n_headers;
1368
1369         /* Node add to tailq. */
1370         TAILQ_INSERT_TAIL(&p->headers, h, node);
1371         p->n_headers++;
1372         p->n_structs++;
1373
1374         return 0;
1375 }
1376
1377 static int
1378 header_build(struct rte_swx_pipeline *p)
1379 {
1380         struct header *h;
1381         uint32_t n_bytes = 0, i;
1382
1383         TAILQ_FOREACH(h, &p->headers, node) {
1384                 n_bytes += h->st->n_bits / 8;
1385         }
1386
1387         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1388                 struct thread *t = &p->threads[i];
1389                 uint32_t offset = 0;
1390
1391                 t->headers = calloc(p->n_headers,
1392                                     sizeof(struct header_runtime));
1393                 CHECK(t->headers, ENOMEM);
1394
1395                 t->headers_out = calloc(p->n_headers,
1396                                         sizeof(struct header_out_runtime));
1397                 CHECK(t->headers_out, ENOMEM);
1398
1399                 t->header_storage = calloc(1, n_bytes);
1400                 CHECK(t->header_storage, ENOMEM);
1401
1402                 t->header_out_storage = calloc(1, n_bytes);
1403                 CHECK(t->header_out_storage, ENOMEM);
1404
1405                 TAILQ_FOREACH(h, &p->headers, node) {
1406                         uint8_t *header_storage;
1407                         uint32_t n_bytes =  h->st->n_bits / 8;
1408
1409                         header_storage = &t->header_storage[offset];
1410                         offset += n_bytes;
1411
1412                         t->headers[h->id].ptr0 = header_storage;
1413                         t->headers[h->id].n_bytes = n_bytes;
1414
1415                         t->structs[h->struct_id] = header_storage;
1416                 }
1417         }
1418
1419         return 0;
1420 }
1421
1422 static void
1423 header_build_free(struct rte_swx_pipeline *p)
1424 {
1425         uint32_t i;
1426
1427         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1428                 struct thread *t = &p->threads[i];
1429
1430                 free(t->headers_out);
1431                 t->headers_out = NULL;
1432
1433                 free(t->headers);
1434                 t->headers = NULL;
1435
1436                 free(t->header_out_storage);
1437                 t->header_out_storage = NULL;
1438
1439                 free(t->header_storage);
1440                 t->header_storage = NULL;
1441         }
1442 }
1443
1444 static void
1445 header_free(struct rte_swx_pipeline *p)
1446 {
1447         header_build_free(p);
1448
1449         for ( ; ; ) {
1450                 struct header *elem;
1451
1452                 elem = TAILQ_FIRST(&p->headers);
1453                 if (!elem)
1454                         break;
1455
1456                 TAILQ_REMOVE(&p->headers, elem, node);
1457                 free(elem);
1458         }
1459 }
1460
1461 /*
1462  * Meta-data.
1463  */
1464 static struct field *
1465 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1466 {
1467         if (!p->metadata_st)
1468                 return NULL;
1469
1470         if (name[0] != 'm' || name[1] != '.')
1471                 return NULL;
1472
1473         return struct_type_field_find(p->metadata_st, &name[2]);
1474 }
1475
1476 int
1477 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1478                                           const char *struct_type_name)
1479 {
1480         struct struct_type *st = NULL;
1481
1482         CHECK(p, EINVAL);
1483
1484         CHECK_NAME(struct_type_name, EINVAL);
1485         st  = struct_type_find(p, struct_type_name);
1486         CHECK(st, EINVAL);
1487         CHECK(!st->var_size, EINVAL);
1488         CHECK(!p->metadata_st, EINVAL);
1489
1490         p->metadata_st = st;
1491         p->metadata_struct_id = p->n_structs;
1492
1493         p->n_structs++;
1494
1495         return 0;
1496 }
1497
1498 static int
1499 metadata_build(struct rte_swx_pipeline *p)
1500 {
1501         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1502         uint32_t i;
1503
1504         /* Thread-level initialization. */
1505         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1506                 struct thread *t = &p->threads[i];
1507                 uint8_t *metadata;
1508
1509                 metadata = calloc(1, n_bytes);
1510                 CHECK(metadata, ENOMEM);
1511
1512                 t->metadata = metadata;
1513                 t->structs[p->metadata_struct_id] = metadata;
1514         }
1515
1516         return 0;
1517 }
1518
1519 static void
1520 metadata_build_free(struct rte_swx_pipeline *p)
1521 {
1522         uint32_t i;
1523
1524         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1525                 struct thread *t = &p->threads[i];
1526
1527                 free(t->metadata);
1528                 t->metadata = NULL;
1529         }
1530 }
1531
1532 static void
1533 metadata_free(struct rte_swx_pipeline *p)
1534 {
1535         metadata_build_free(p);
1536 }
1537
1538 /*
1539  * Instruction.
1540  */
1541 static int
1542 instruction_is_tx(enum instruction_type type)
1543 {
1544         switch (type) {
1545         case INSTR_TX:
1546         case INSTR_TX_I:
1547         case INSTR_DROP:
1548                 return 1;
1549
1550         default:
1551                 return 0;
1552         }
1553 }
1554
1555 static int
1556 instruction_does_tx(struct instruction *instr)
1557 {
1558         switch (instr->type) {
1559         case INSTR_TX:
1560         case INSTR_TX_I:
1561         case INSTR_DROP:
1562         case INSTR_HDR_EMIT_TX:
1563         case INSTR_HDR_EMIT2_TX:
1564         case INSTR_HDR_EMIT3_TX:
1565         case INSTR_HDR_EMIT4_TX:
1566         case INSTR_HDR_EMIT5_TX:
1567         case INSTR_HDR_EMIT6_TX:
1568         case INSTR_HDR_EMIT7_TX:
1569         case INSTR_HDR_EMIT8_TX:
1570                 return 1;
1571         default:
1572                 return 0;
1573         }
1574 }
1575
1576 static int
1577 instruction_is_jmp(struct instruction *instr)
1578 {
1579         switch (instr->type) {
1580         case INSTR_JMP:
1581         case INSTR_JMP_VALID:
1582         case INSTR_JMP_INVALID:
1583         case INSTR_JMP_HIT:
1584         case INSTR_JMP_MISS:
1585         case INSTR_JMP_ACTION_HIT:
1586         case INSTR_JMP_ACTION_MISS:
1587         case INSTR_JMP_EQ:
1588         case INSTR_JMP_EQ_MH:
1589         case INSTR_JMP_EQ_HM:
1590         case INSTR_JMP_EQ_HH:
1591         case INSTR_JMP_EQ_I:
1592         case INSTR_JMP_NEQ:
1593         case INSTR_JMP_NEQ_MH:
1594         case INSTR_JMP_NEQ_HM:
1595         case INSTR_JMP_NEQ_HH:
1596         case INSTR_JMP_NEQ_I:
1597         case INSTR_JMP_LT:
1598         case INSTR_JMP_LT_MH:
1599         case INSTR_JMP_LT_HM:
1600         case INSTR_JMP_LT_HH:
1601         case INSTR_JMP_LT_MI:
1602         case INSTR_JMP_LT_HI:
1603         case INSTR_JMP_GT:
1604         case INSTR_JMP_GT_MH:
1605         case INSTR_JMP_GT_HM:
1606         case INSTR_JMP_GT_HH:
1607         case INSTR_JMP_GT_MI:
1608         case INSTR_JMP_GT_HI:
1609                 return 1;
1610
1611         default:
1612                 return 0;
1613         }
1614 }
1615
1616 static int
1617 instruction_does_thread_yield(struct instruction *instr)
1618 {
1619         switch (instr->type) {
1620         case INSTR_RX:
1621         case INSTR_TABLE:
1622         case INSTR_TABLE_AF:
1623         case INSTR_SELECTOR:
1624         case INSTR_LEARNER:
1625         case INSTR_LEARNER_AF:
1626         case INSTR_EXTERN_OBJ:
1627         case INSTR_EXTERN_FUNC:
1628                 return 1;
1629         default:
1630                 return 0;
1631         }
1632 }
1633
1634 static struct field *
1635 action_field_parse(struct action *action, const char *name);
1636
1637 static struct field *
1638 struct_field_parse(struct rte_swx_pipeline *p,
1639                    struct action *action,
1640                    const char *name,
1641                    uint32_t *struct_id)
1642 {
1643         struct field *f;
1644
1645         switch (name[0]) {
1646         case 'h':
1647         {
1648                 struct header *header;
1649
1650                 f = header_field_parse(p, name, &header);
1651                 if (!f)
1652                         return NULL;
1653
1654                 *struct_id = header->struct_id;
1655                 return f;
1656         }
1657
1658         case 'm':
1659         {
1660                 f = metadata_field_parse(p, name);
1661                 if (!f)
1662                         return NULL;
1663
1664                 *struct_id = p->metadata_struct_id;
1665                 return f;
1666         }
1667
1668         case 't':
1669         {
1670                 if (!action)
1671                         return NULL;
1672
1673                 f = action_field_parse(action, name);
1674                 if (!f)
1675                         return NULL;
1676
1677                 *struct_id = 0;
1678                 return f;
1679         }
1680
1681         case 'e':
1682         {
1683                 struct extern_obj *obj;
1684
1685                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1686                 if (!f)
1687                         return NULL;
1688
1689                 *struct_id = obj->struct_id;
1690                 return f;
1691         }
1692
1693         case 'f':
1694         {
1695                 struct extern_func *func;
1696
1697                 f = extern_func_mailbox_field_parse(p, name, &func);
1698                 if (!f)
1699                         return NULL;
1700
1701                 *struct_id = func->struct_id;
1702                 return f;
1703         }
1704
1705         default:
1706                 return NULL;
1707         }
1708 }
1709
1710 /*
1711  * rx.
1712  */
1713 static int
1714 instr_rx_translate(struct rte_swx_pipeline *p,
1715                    struct action *action,
1716                    char **tokens,
1717                    int n_tokens,
1718                    struct instruction *instr,
1719                    struct instruction_data *data __rte_unused)
1720 {
1721         struct field *f;
1722
1723         CHECK(!action, EINVAL);
1724         CHECK(n_tokens == 2, EINVAL);
1725
1726         f = metadata_field_parse(p, tokens[1]);
1727         CHECK(f, EINVAL);
1728
1729         instr->type = INSTR_RX;
1730         instr->io.io.offset = f->offset / 8;
1731         instr->io.io.n_bits = f->n_bits;
1732         return 0;
1733 }
1734
1735 /*
1736  * tx.
1737  */
1738 static int
1739 instr_tx_translate(struct rte_swx_pipeline *p,
1740                    struct action *action __rte_unused,
1741                    char **tokens,
1742                    int n_tokens,
1743                    struct instruction *instr,
1744                    struct instruction_data *data __rte_unused)
1745 {
1746         char *port = tokens[1];
1747         struct field *f;
1748         uint32_t port_val;
1749
1750         CHECK(n_tokens == 2, EINVAL);
1751
1752         f = metadata_field_parse(p, port);
1753         if (f) {
1754                 instr->type = INSTR_TX;
1755                 instr->io.io.offset = f->offset / 8;
1756                 instr->io.io.n_bits = f->n_bits;
1757                 return 0;
1758         }
1759
1760         /* TX_I. */
1761         port_val = strtoul(port, &port, 0);
1762         CHECK(!port[0], EINVAL);
1763
1764         instr->type = INSTR_TX_I;
1765         instr->io.io.val = port_val;
1766         return 0;
1767 }
1768
1769 static int
1770 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1771                      struct action *action __rte_unused,
1772                      char **tokens __rte_unused,
1773                      int n_tokens,
1774                      struct instruction *instr,
1775                      struct instruction_data *data __rte_unused)
1776 {
1777         CHECK(n_tokens == 1, EINVAL);
1778
1779         /* DROP. */
1780         instr->type = INSTR_DROP;
1781         return 0;
1782 }
1783
1784 static inline void
1785 instr_tx_exec(struct rte_swx_pipeline *p)
1786 {
1787         struct thread *t = &p->threads[p->thread_id];
1788         struct instruction *ip = t->ip;
1789
1790         __instr_tx_exec(p, t, ip);
1791
1792         /* Thread. */
1793         thread_ip_reset(p, t);
1794         instr_rx_exec(p);
1795 }
1796
1797 static inline void
1798 instr_tx_i_exec(struct rte_swx_pipeline *p)
1799 {
1800         struct thread *t = &p->threads[p->thread_id];
1801         struct instruction *ip = t->ip;
1802
1803         __instr_tx_i_exec(p, t, ip);
1804
1805         /* Thread. */
1806         thread_ip_reset(p, t);
1807         instr_rx_exec(p);
1808 }
1809
1810 static inline void
1811 instr_drop_exec(struct rte_swx_pipeline *p)
1812 {
1813         struct thread *t = &p->threads[p->thread_id];
1814         struct instruction *ip = t->ip;
1815
1816         __instr_drop_exec(p, t, ip);
1817
1818         /* Thread. */
1819         thread_ip_reset(p, t);
1820         instr_rx_exec(p);
1821 }
1822
1823 /*
1824  * mirror.
1825  */
1826 static int
1827 instr_mirror_translate(struct rte_swx_pipeline *p,
1828                        struct action *action,
1829                        char **tokens,
1830                        int n_tokens,
1831                        struct instruction *instr,
1832                        struct instruction_data *data __rte_unused)
1833 {
1834         char *dst = tokens[1], *src = tokens[2];
1835         struct field *fdst, *fsrc;
1836         uint32_t dst_struct_id = 0, src_struct_id = 0;
1837
1838         CHECK(n_tokens == 3, EINVAL);
1839
1840         fdst = struct_field_parse(p, action, dst, &dst_struct_id);
1841         CHECK(fdst, EINVAL);
1842         CHECK(dst[0] != 'h', EINVAL);
1843         CHECK(!fdst->var_size, EINVAL);
1844
1845         fsrc = struct_field_parse(p, action, src, &src_struct_id);
1846         CHECK(fsrc, EINVAL);
1847         CHECK(src[0] != 'h', EINVAL);
1848         CHECK(!fsrc->var_size, EINVAL);
1849
1850         instr->type = INSTR_MIRROR;
1851         instr->mirror.dst.struct_id = (uint8_t)dst_struct_id;
1852         instr->mirror.dst.n_bits = fdst->n_bits;
1853         instr->mirror.dst.offset = fdst->offset / 8;
1854         instr->mirror.src.struct_id = (uint8_t)src_struct_id;
1855         instr->mirror.src.n_bits = fsrc->n_bits;
1856         instr->mirror.src.offset = fsrc->offset / 8;
1857
1858         return 0;
1859 }
1860
1861 static inline void
1862 instr_mirror_exec(struct rte_swx_pipeline *p)
1863 {
1864         struct thread *t = &p->threads[p->thread_id];
1865         struct instruction *ip = t->ip;
1866
1867         __instr_mirror_exec(p, t, ip);
1868
1869         /* Thread. */
1870         thread_ip_inc(p);
1871 }
1872
1873 /*
1874  * recirculate.
1875  */
1876 static int
1877 instr_recirculate_translate(struct rte_swx_pipeline *p __rte_unused,
1878                             struct action *action __rte_unused,
1879                             char **tokens __rte_unused,
1880                             int n_tokens,
1881                             struct instruction *instr,
1882                             struct instruction_data *data __rte_unused)
1883 {
1884         CHECK(n_tokens == 1, EINVAL);
1885
1886         instr->type = INSTR_RECIRCULATE;
1887         return 0;
1888 }
1889
1890 static int
1891 instr_recircid_translate(struct rte_swx_pipeline *p,
1892                          struct action *action __rte_unused,
1893                          char **tokens,
1894                          int n_tokens,
1895                          struct instruction *instr,
1896                          struct instruction_data *data __rte_unused)
1897 {
1898         struct field *f;
1899
1900         CHECK(n_tokens == 2, EINVAL);
1901
1902         f = metadata_field_parse(p, tokens[1]);
1903         CHECK(f, EINVAL);
1904
1905         instr->type = INSTR_RECIRCID;
1906         instr->io.io.offset = f->offset / 8;
1907         instr->io.io.n_bits = f->n_bits;
1908         return 0;
1909 }
1910
1911 static inline void
1912 instr_recirculate_exec(struct rte_swx_pipeline *p)
1913 {
1914         struct thread *t = &p->threads[p->thread_id];
1915         struct instruction *ip = t->ip;
1916
1917         __instr_recirculate_exec(p, t, ip);
1918
1919         /* Thread. */
1920         thread_ip_inc(p);
1921 }
1922
1923 static inline void
1924 instr_recircid_exec(struct rte_swx_pipeline *p)
1925 {
1926         struct thread *t = &p->threads[p->thread_id];
1927         struct instruction *ip = t->ip;
1928
1929         __instr_recircid_exec(p, t, ip);
1930
1931         /* Thread. */
1932         thread_ip_inc(p);
1933 }
1934
1935 /*
1936  * extract.
1937  */
1938 static int
1939 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1940                             struct action *action,
1941                             char **tokens,
1942                             int n_tokens,
1943                             struct instruction *instr,
1944                             struct instruction_data *data __rte_unused)
1945 {
1946         struct header *h;
1947
1948         CHECK(!action, EINVAL);
1949         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1950
1951         h = header_parse(p, tokens[1]);
1952         CHECK(h, EINVAL);
1953
1954         if (n_tokens == 2) {
1955                 CHECK(!h->st->var_size, EINVAL);
1956
1957                 instr->type = INSTR_HDR_EXTRACT;
1958                 instr->io.hdr.header_id[0] = h->id;
1959                 instr->io.hdr.struct_id[0] = h->struct_id;
1960                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1961         } else {
1962                 struct field *mf;
1963
1964                 CHECK(h->st->var_size, EINVAL);
1965
1966                 mf = metadata_field_parse(p, tokens[2]);
1967                 CHECK(mf, EINVAL);
1968                 CHECK(!mf->var_size, EINVAL);
1969
1970                 instr->type = INSTR_HDR_EXTRACT_M;
1971                 instr->io.io.offset = mf->offset / 8;
1972                 instr->io.io.n_bits = mf->n_bits;
1973                 instr->io.hdr.header_id[0] = h->id;
1974                 instr->io.hdr.struct_id[0] = h->struct_id;
1975                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1976         }
1977
1978         return 0;
1979 }
1980
1981 static int
1982 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1983                               struct action *action,
1984                               char **tokens,
1985                               int n_tokens,
1986                               struct instruction *instr,
1987                               struct instruction_data *data __rte_unused)
1988 {
1989         struct header *h;
1990
1991         CHECK(!action, EINVAL);
1992         CHECK(n_tokens == 2, EINVAL);
1993
1994         h = header_parse(p, tokens[1]);
1995         CHECK(h, EINVAL);
1996         CHECK(!h->st->var_size, EINVAL);
1997
1998         instr->type = INSTR_HDR_LOOKAHEAD;
1999         instr->io.hdr.header_id[0] = h->id;
2000         instr->io.hdr.struct_id[0] = h->struct_id;
2001         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
2002
2003         return 0;
2004 }
2005
2006 static inline void
2007 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2008 {
2009         struct thread *t = &p->threads[p->thread_id];
2010         struct instruction *ip = t->ip;
2011
2012         __instr_hdr_extract_exec(p, t, ip);
2013
2014         /* Thread. */
2015         thread_ip_inc(p);
2016 }
2017
2018 static inline void
2019 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2020 {
2021         struct thread *t = &p->threads[p->thread_id];
2022         struct instruction *ip = t->ip;
2023
2024         __instr_hdr_extract2_exec(p, t, ip);
2025
2026         /* Thread. */
2027         thread_ip_inc(p);
2028 }
2029
2030 static inline void
2031 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2032 {
2033         struct thread *t = &p->threads[p->thread_id];
2034         struct instruction *ip = t->ip;
2035
2036         __instr_hdr_extract3_exec(p, t, ip);
2037
2038         /* Thread. */
2039         thread_ip_inc(p);
2040 }
2041
2042 static inline void
2043 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2044 {
2045         struct thread *t = &p->threads[p->thread_id];
2046         struct instruction *ip = t->ip;
2047
2048         __instr_hdr_extract4_exec(p, t, ip);
2049
2050         /* Thread. */
2051         thread_ip_inc(p);
2052 }
2053
2054 static inline void
2055 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2056 {
2057         struct thread *t = &p->threads[p->thread_id];
2058         struct instruction *ip = t->ip;
2059
2060         __instr_hdr_extract5_exec(p, t, ip);
2061
2062         /* Thread. */
2063         thread_ip_inc(p);
2064 }
2065
2066 static inline void
2067 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2068 {
2069         struct thread *t = &p->threads[p->thread_id];
2070         struct instruction *ip = t->ip;
2071
2072         __instr_hdr_extract6_exec(p, t, ip);
2073
2074         /* Thread. */
2075         thread_ip_inc(p);
2076 }
2077
2078 static inline void
2079 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2080 {
2081         struct thread *t = &p->threads[p->thread_id];
2082         struct instruction *ip = t->ip;
2083
2084         __instr_hdr_extract7_exec(p, t, ip);
2085
2086         /* Thread. */
2087         thread_ip_inc(p);
2088 }
2089
2090 static inline void
2091 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2092 {
2093         struct thread *t = &p->threads[p->thread_id];
2094         struct instruction *ip = t->ip;
2095
2096         __instr_hdr_extract8_exec(p, t, ip);
2097
2098         /* Thread. */
2099         thread_ip_inc(p);
2100 }
2101
2102 static inline void
2103 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
2104 {
2105         struct thread *t = &p->threads[p->thread_id];
2106         struct instruction *ip = t->ip;
2107
2108         __instr_hdr_extract_m_exec(p, t, ip);
2109
2110         /* Thread. */
2111         thread_ip_inc(p);
2112 }
2113
2114 static inline void
2115 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
2116 {
2117         struct thread *t = &p->threads[p->thread_id];
2118         struct instruction *ip = t->ip;
2119
2120         __instr_hdr_lookahead_exec(p, t, ip);
2121
2122         /* Thread. */
2123         thread_ip_inc(p);
2124 }
2125
2126 /*
2127  * emit.
2128  */
2129 static int
2130 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2131                          struct action *action __rte_unused,
2132                          char **tokens,
2133                          int n_tokens,
2134                          struct instruction *instr,
2135                          struct instruction_data *data __rte_unused)
2136 {
2137         struct header *h;
2138
2139         CHECK(n_tokens == 2, EINVAL);
2140
2141         h = header_parse(p, tokens[1]);
2142         CHECK(h, EINVAL);
2143
2144         instr->type = INSTR_HDR_EMIT;
2145         instr->io.hdr.header_id[0] = h->id;
2146         instr->io.hdr.struct_id[0] = h->struct_id;
2147         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2148         return 0;
2149 }
2150
2151 static inline void
2152 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2153 {
2154         struct thread *t = &p->threads[p->thread_id];
2155         struct instruction *ip = t->ip;
2156
2157         __instr_hdr_emit_exec(p, t, ip);
2158
2159         /* Thread. */
2160         thread_ip_inc(p);
2161 }
2162
2163 static inline void
2164 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2165 {
2166         struct thread *t = &p->threads[p->thread_id];
2167         struct instruction *ip = t->ip;
2168
2169         __instr_hdr_emit_tx_exec(p, t, ip);
2170
2171         /* Thread. */
2172         thread_ip_reset(p, t);
2173         instr_rx_exec(p);
2174 }
2175
2176 static inline void
2177 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2178 {
2179         struct thread *t = &p->threads[p->thread_id];
2180         struct instruction *ip = t->ip;
2181
2182         __instr_hdr_emit2_tx_exec(p, t, ip);
2183
2184         /* Thread. */
2185         thread_ip_reset(p, t);
2186         instr_rx_exec(p);
2187 }
2188
2189 static inline void
2190 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2191 {
2192         struct thread *t = &p->threads[p->thread_id];
2193         struct instruction *ip = t->ip;
2194
2195         __instr_hdr_emit3_tx_exec(p, t, ip);
2196
2197         /* Thread. */
2198         thread_ip_reset(p, t);
2199         instr_rx_exec(p);
2200 }
2201
2202 static inline void
2203 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2204 {
2205         struct thread *t = &p->threads[p->thread_id];
2206         struct instruction *ip = t->ip;
2207
2208         __instr_hdr_emit4_tx_exec(p, t, ip);
2209
2210         /* Thread. */
2211         thread_ip_reset(p, t);
2212         instr_rx_exec(p);
2213 }
2214
2215 static inline void
2216 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2217 {
2218         struct thread *t = &p->threads[p->thread_id];
2219         struct instruction *ip = t->ip;
2220
2221         __instr_hdr_emit5_tx_exec(p, t, ip);
2222
2223         /* Thread. */
2224         thread_ip_reset(p, t);
2225         instr_rx_exec(p);
2226 }
2227
2228 static inline void
2229 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2230 {
2231         struct thread *t = &p->threads[p->thread_id];
2232         struct instruction *ip = t->ip;
2233
2234         __instr_hdr_emit6_tx_exec(p, t, ip);
2235
2236         /* Thread. */
2237         thread_ip_reset(p, t);
2238         instr_rx_exec(p);
2239 }
2240
2241 static inline void
2242 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2243 {
2244         struct thread *t = &p->threads[p->thread_id];
2245         struct instruction *ip = t->ip;
2246
2247         __instr_hdr_emit7_tx_exec(p, t, ip);
2248
2249         /* Thread. */
2250         thread_ip_reset(p, t);
2251         instr_rx_exec(p);
2252 }
2253
2254 static inline void
2255 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2256 {
2257         struct thread *t = &p->threads[p->thread_id];
2258         struct instruction *ip = t->ip;
2259
2260         __instr_hdr_emit8_tx_exec(p, t, ip);
2261
2262         /* Thread. */
2263         thread_ip_reset(p, t);
2264         instr_rx_exec(p);
2265 }
2266
2267 /*
2268  * validate.
2269  */
2270 static int
2271 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2272                              struct action *action __rte_unused,
2273                              char **tokens,
2274                              int n_tokens,
2275                              struct instruction *instr,
2276                              struct instruction_data *data __rte_unused)
2277 {
2278         struct header *h;
2279
2280         CHECK(n_tokens == 2, EINVAL);
2281
2282         h = header_parse(p, tokens[1]);
2283         CHECK(h, EINVAL);
2284
2285         instr->type = INSTR_HDR_VALIDATE;
2286         instr->valid.header_id = h->id;
2287         return 0;
2288 }
2289
2290 static inline void
2291 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2292 {
2293         struct thread *t = &p->threads[p->thread_id];
2294         struct instruction *ip = t->ip;
2295
2296         __instr_hdr_validate_exec(p, t, ip);
2297
2298         /* Thread. */
2299         thread_ip_inc(p);
2300 }
2301
2302 /*
2303  * invalidate.
2304  */
2305 static int
2306 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2307                                struct action *action __rte_unused,
2308                                char **tokens,
2309                                int n_tokens,
2310                                struct instruction *instr,
2311                                struct instruction_data *data __rte_unused)
2312 {
2313         struct header *h;
2314
2315         CHECK(n_tokens == 2, EINVAL);
2316
2317         h = header_parse(p, tokens[1]);
2318         CHECK(h, EINVAL);
2319
2320         instr->type = INSTR_HDR_INVALIDATE;
2321         instr->valid.header_id = h->id;
2322         return 0;
2323 }
2324
2325 static inline void
2326 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2327 {
2328         struct thread *t = &p->threads[p->thread_id];
2329         struct instruction *ip = t->ip;
2330
2331         __instr_hdr_invalidate_exec(p, t, ip);
2332
2333         /* Thread. */
2334         thread_ip_inc(p);
2335 }
2336
2337 /*
2338  * table.
2339  */
2340 static struct table *
2341 table_find(struct rte_swx_pipeline *p, const char *name);
2342
2343 static struct selector *
2344 selector_find(struct rte_swx_pipeline *p, const char *name);
2345
2346 static struct learner *
2347 learner_find(struct rte_swx_pipeline *p, const char *name);
2348
2349 static int
2350 instr_table_translate(struct rte_swx_pipeline *p,
2351                       struct action *action,
2352                       char **tokens,
2353                       int n_tokens,
2354                       struct instruction *instr,
2355                       struct instruction_data *data __rte_unused)
2356 {
2357         struct table *t;
2358         struct selector *s;
2359         struct learner *l;
2360
2361         CHECK(!action, EINVAL);
2362         CHECK(n_tokens == 2, EINVAL);
2363
2364         t = table_find(p, tokens[1]);
2365         if (t) {
2366                 instr->type = INSTR_TABLE;
2367                 instr->table.table_id = t->id;
2368                 return 0;
2369         }
2370
2371         s = selector_find(p, tokens[1]);
2372         if (s) {
2373                 instr->type = INSTR_SELECTOR;
2374                 instr->table.table_id = s->id;
2375                 return 0;
2376         }
2377
2378         l = learner_find(p, tokens[1]);
2379         if (l) {
2380                 instr->type = INSTR_LEARNER;
2381                 instr->table.table_id = l->id;
2382                 return 0;
2383         }
2384
2385         CHECK(0, EINVAL);
2386 }
2387
2388 static inline void
2389 instr_table_exec(struct rte_swx_pipeline *p)
2390 {
2391         struct thread *t = &p->threads[p->thread_id];
2392         struct instruction *ip = t->ip;
2393         uint32_t table_id = ip->table.table_id;
2394         struct rte_swx_table_state *ts = &t->table_state[table_id];
2395         struct table_runtime *table = &t->tables[table_id];
2396         struct table_statistics *stats = &p->table_stats[table_id];
2397         uint64_t action_id, n_pkts_hit, n_pkts_action;
2398         uint8_t *action_data;
2399         int done, hit;
2400
2401         /* Table. */
2402         done = table->func(ts->obj,
2403                            table->mailbox,
2404                            table->key,
2405                            &action_id,
2406                            &action_data,
2407                            &hit);
2408         if (!done) {
2409                 /* Thread. */
2410                 TRACE("[Thread %2u] table %u (not finalized)\n",
2411                       p->thread_id,
2412                       table_id);
2413
2414                 thread_yield(p);
2415                 return;
2416         }
2417
2418         action_id = hit ? action_id : ts->default_action_id;
2419         action_data = hit ? action_data : ts->default_action_data;
2420         n_pkts_hit = stats->n_pkts_hit[hit];
2421         n_pkts_action = stats->n_pkts_action[action_id];
2422
2423         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2424               p->thread_id,
2425               table_id,
2426               hit ? "hit" : "miss",
2427               (uint32_t)action_id);
2428
2429         t->action_id = action_id;
2430         t->structs[0] = action_data;
2431         t->hit = hit;
2432         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2433         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2434
2435         /* Thread. */
2436         thread_ip_action_call(p, t, action_id);
2437 }
2438
2439 static inline void
2440 instr_table_af_exec(struct rte_swx_pipeline *p)
2441 {
2442         struct thread *t = &p->threads[p->thread_id];
2443         struct instruction *ip = t->ip;
2444         uint32_t table_id = ip->table.table_id;
2445         struct rte_swx_table_state *ts = &t->table_state[table_id];
2446         struct table_runtime *table = &t->tables[table_id];
2447         struct table_statistics *stats = &p->table_stats[table_id];
2448         uint64_t action_id, n_pkts_hit, n_pkts_action;
2449         uint8_t *action_data;
2450         action_func_t action_func;
2451         int done, hit;
2452
2453         /* Table. */
2454         done = table->func(ts->obj,
2455                            table->mailbox,
2456                            table->key,
2457                            &action_id,
2458                            &action_data,
2459                            &hit);
2460         if (!done) {
2461                 /* Thread. */
2462                 TRACE("[Thread %2u] table %u (not finalized)\n",
2463                       p->thread_id,
2464                       table_id);
2465
2466                 thread_yield(p);
2467                 return;
2468         }
2469
2470         action_id = hit ? action_id : ts->default_action_id;
2471         action_data = hit ? action_data : ts->default_action_data;
2472         action_func = p->action_funcs[action_id];
2473         n_pkts_hit = stats->n_pkts_hit[hit];
2474         n_pkts_action = stats->n_pkts_action[action_id];
2475
2476         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2477               p->thread_id,
2478               table_id,
2479               hit ? "hit" : "miss",
2480               (uint32_t)action_id);
2481
2482         t->action_id = action_id;
2483         t->structs[0] = action_data;
2484         t->hit = hit;
2485         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2486         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2487
2488         /* Thread. */
2489         thread_ip_inc(p);
2490
2491         /* Action. */
2492         action_func(p);
2493 }
2494
2495 static inline void
2496 instr_selector_exec(struct rte_swx_pipeline *p)
2497 {
2498         struct thread *t = &p->threads[p->thread_id];
2499         struct instruction *ip = t->ip;
2500         uint32_t selector_id = ip->table.table_id;
2501         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2502         struct selector_runtime *selector = &t->selectors[selector_id];
2503         struct selector_statistics *stats = &p->selector_stats[selector_id];
2504         uint64_t n_pkts = stats->n_pkts;
2505         int done;
2506
2507         /* Table. */
2508         done = rte_swx_table_selector_select(ts->obj,
2509                            selector->mailbox,
2510                            selector->group_id_buffer,
2511                            selector->selector_buffer,
2512                            selector->member_id_buffer);
2513         if (!done) {
2514                 /* Thread. */
2515                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2516                       p->thread_id,
2517                       selector_id);
2518
2519                 thread_yield(p);
2520                 return;
2521         }
2522
2523
2524         TRACE("[Thread %2u] selector %u\n",
2525               p->thread_id,
2526               selector_id);
2527
2528         stats->n_pkts = n_pkts + 1;
2529
2530         /* Thread. */
2531         thread_ip_inc(p);
2532 }
2533
2534 static inline void
2535 instr_learner_exec(struct rte_swx_pipeline *p)
2536 {
2537         struct thread *t = &p->threads[p->thread_id];
2538         struct instruction *ip = t->ip;
2539         uint32_t learner_id = ip->table.table_id;
2540         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2541                 p->n_selectors + learner_id];
2542         struct learner_runtime *l = &t->learners[learner_id];
2543         struct learner_statistics *stats = &p->learner_stats[learner_id];
2544         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2545         uint8_t *action_data;
2546         int done, hit;
2547
2548         /* Table. */
2549         time = rte_get_tsc_cycles();
2550
2551         done = rte_swx_table_learner_lookup(ts->obj,
2552                                             l->mailbox,
2553                                             time,
2554                                             l->key,
2555                                             &action_id,
2556                                             &action_data,
2557                                             &hit);
2558         if (!done) {
2559                 /* Thread. */
2560                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2561                       p->thread_id,
2562                       learner_id);
2563
2564                 thread_yield(p);
2565                 return;
2566         }
2567
2568         action_id = hit ? action_id : ts->default_action_id;
2569         action_data = hit ? action_data : ts->default_action_data;
2570         n_pkts_hit = stats->n_pkts_hit[hit];
2571         n_pkts_action = stats->n_pkts_action[action_id];
2572
2573         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2574               p->thread_id,
2575               learner_id,
2576               hit ? "hit" : "miss",
2577               (uint32_t)action_id);
2578
2579         t->action_id = action_id;
2580         t->structs[0] = action_data;
2581         t->hit = hit;
2582         t->learner_id = learner_id;
2583         t->time = time;
2584         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2585         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2586
2587         /* Thread. */
2588         thread_ip_action_call(p, t, action_id);
2589 }
2590
2591 static inline void
2592 instr_learner_af_exec(struct rte_swx_pipeline *p)
2593 {
2594         struct thread *t = &p->threads[p->thread_id];
2595         struct instruction *ip = t->ip;
2596         uint32_t learner_id = ip->table.table_id;
2597         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2598                 p->n_selectors + learner_id];
2599         struct learner_runtime *l = &t->learners[learner_id];
2600         struct learner_statistics *stats = &p->learner_stats[learner_id];
2601         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2602         uint8_t *action_data;
2603         action_func_t action_func;
2604         int done, hit;
2605
2606         /* Table. */
2607         time = rte_get_tsc_cycles();
2608
2609         done = rte_swx_table_learner_lookup(ts->obj,
2610                                             l->mailbox,
2611                                             time,
2612                                             l->key,
2613                                             &action_id,
2614                                             &action_data,
2615                                             &hit);
2616         if (!done) {
2617                 /* Thread. */
2618                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2619                       p->thread_id,
2620                       learner_id);
2621
2622                 thread_yield(p);
2623                 return;
2624         }
2625
2626         action_id = hit ? action_id : ts->default_action_id;
2627         action_data = hit ? action_data : ts->default_action_data;
2628         action_func = p->action_funcs[action_id];
2629         n_pkts_hit = stats->n_pkts_hit[hit];
2630         n_pkts_action = stats->n_pkts_action[action_id];
2631
2632         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2633               p->thread_id,
2634               learner_id,
2635               hit ? "hit" : "miss",
2636               (uint32_t)action_id);
2637
2638         t->action_id = action_id;
2639         t->structs[0] = action_data;
2640         t->hit = hit;
2641         t->learner_id = learner_id;
2642         t->time = time;
2643         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2644         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2645
2646         /* Thread. */
2647         thread_ip_inc(p);
2648
2649         /* Action */
2650         action_func(p);
2651 }
2652
2653 /*
2654  * learn.
2655  */
2656 static struct action *
2657 action_find(struct rte_swx_pipeline *p, const char *name);
2658
2659 static int
2660 action_has_nbo_args(struct action *a);
2661
2662 static int
2663 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2664
2665 static int
2666 instr_learn_translate(struct rte_swx_pipeline *p,
2667                       struct action *action,
2668                       char **tokens,
2669                       int n_tokens,
2670                       struct instruction *instr,
2671                       struct instruction_data *data __rte_unused)
2672 {
2673         struct action *a;
2674         struct field *mf_first_arg = NULL, *mf_timeout_id = NULL;
2675         const char *mf_first_arg_name, *mf_timeout_id_name;
2676
2677         CHECK(action, EINVAL);
2678         CHECK((n_tokens == 3) || (n_tokens == 4), EINVAL);
2679
2680         /* Action. */
2681         a = action_find(p, tokens[1]);
2682         CHECK(a, EINVAL);
2683         CHECK(!action_has_nbo_args(a), EINVAL);
2684
2685         /* Action first argument. */
2686         mf_first_arg_name = (n_tokens == 4) ? tokens[2] : NULL;
2687         CHECK(!learner_action_args_check(p, a, mf_first_arg_name), EINVAL);
2688
2689         if (mf_first_arg_name) {
2690                 mf_first_arg = metadata_field_parse(p, mf_first_arg_name);
2691                 CHECK(mf_first_arg, EINVAL);
2692         }
2693
2694         /* Timeout ID. */
2695         mf_timeout_id_name = (n_tokens == 4) ? tokens[3] : tokens[2];
2696         CHECK_NAME(mf_timeout_id_name, EINVAL);
2697         mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
2698         CHECK(mf_timeout_id, EINVAL);
2699
2700         /* Instruction. */
2701         instr->type = INSTR_LEARNER_LEARN;
2702         instr->learn.action_id = a->id;
2703         instr->learn.mf_first_arg_offset = mf_first_arg ? (mf_first_arg->offset / 8) : 0;
2704         instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
2705         instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;
2706
2707         return 0;
2708 }
2709
2710 static inline void
2711 instr_learn_exec(struct rte_swx_pipeline *p)
2712 {
2713         struct thread *t = &p->threads[p->thread_id];
2714         struct instruction *ip = t->ip;
2715
2716         __instr_learn_exec(p, t, ip);
2717
2718         /* Thread. */
2719         thread_ip_inc(p);
2720 }
2721
2722 /*
2723  * rearm.
2724  */
2725 static int
2726 instr_rearm_translate(struct rte_swx_pipeline *p,
2727                       struct action *action,
2728                       char **tokens,
2729                       int n_tokens,
2730                       struct instruction *instr,
2731                       struct instruction_data *data __rte_unused)
2732 {
2733         struct field *mf_timeout_id;
2734         const char *mf_timeout_id_name;
2735
2736         CHECK(action, EINVAL);
2737         CHECK((n_tokens == 1) || (n_tokens == 2), EINVAL);
2738
2739         /* INSTR_LEARNER_REARM. */
2740         if (n_tokens == 1) {
2741                 instr->type = INSTR_LEARNER_REARM;
2742                 return 0;
2743         }
2744
2745         /* INSTR_LEARNER_REARM_NEW. */
2746         mf_timeout_id_name = tokens[1];
2747         CHECK_NAME(mf_timeout_id_name, EINVAL);
2748         mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
2749         CHECK(mf_timeout_id, EINVAL);
2750
2751         instr->type = INSTR_LEARNER_REARM_NEW;
2752         instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
2753         instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;
2754
2755         return 0;
2756 }
2757
2758 static inline void
2759 instr_rearm_exec(struct rte_swx_pipeline *p)
2760 {
2761         struct thread *t = &p->threads[p->thread_id];
2762         struct instruction *ip = t->ip;
2763
2764         __instr_rearm_exec(p, t, ip);
2765
2766         /* Thread. */
2767         thread_ip_inc(p);
2768 }
2769
2770 static inline void
2771 instr_rearm_new_exec(struct rte_swx_pipeline *p)
2772 {
2773         struct thread *t = &p->threads[p->thread_id];
2774         struct instruction *ip = t->ip;
2775
2776         __instr_rearm_new_exec(p, t, ip);
2777
2778         /* Thread. */
2779         thread_ip_inc(p);
2780 }
2781
2782 /*
2783  * forget.
2784  */
2785 static int
2786 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2787                        struct action *action,
2788                        char **tokens __rte_unused,
2789                        int n_tokens,
2790                        struct instruction *instr,
2791                        struct instruction_data *data __rte_unused)
2792 {
2793         CHECK(action, EINVAL);
2794         CHECK(n_tokens == 1, EINVAL);
2795
2796         instr->type = INSTR_LEARNER_FORGET;
2797
2798         return 0;
2799 }
2800
2801 static inline void
2802 instr_forget_exec(struct rte_swx_pipeline *p)
2803 {
2804         struct thread *t = &p->threads[p->thread_id];
2805         struct instruction *ip = t->ip;
2806
2807         __instr_forget_exec(p, t, ip);
2808
2809         /* Thread. */
2810         thread_ip_inc(p);
2811 }
2812
2813 /*
2814  * extern.
2815  */
2816 static int
2817 instr_extern_translate(struct rte_swx_pipeline *p,
2818                        struct action *action __rte_unused,
2819                        char **tokens,
2820                        int n_tokens,
2821                        struct instruction *instr,
2822                        struct instruction_data *data __rte_unused)
2823 {
2824         char *token = tokens[1];
2825
2826         CHECK(n_tokens == 2, EINVAL);
2827
2828         if (token[0] == 'e') {
2829                 struct extern_obj *obj;
2830                 struct extern_type_member_func *func;
2831
2832                 func = extern_obj_member_func_parse(p, token, &obj);
2833                 CHECK(func, EINVAL);
2834
2835                 instr->type = INSTR_EXTERN_OBJ;
2836                 instr->ext_obj.ext_obj_id = obj->id;
2837                 instr->ext_obj.func_id = func->id;
2838
2839                 return 0;
2840         }
2841
2842         if (token[0] == 'f') {
2843                 struct extern_func *func;
2844
2845                 func = extern_func_parse(p, token);
2846                 CHECK(func, EINVAL);
2847
2848                 instr->type = INSTR_EXTERN_FUNC;
2849                 instr->ext_func.ext_func_id = func->id;
2850
2851                 return 0;
2852         }
2853
2854         CHECK(0, EINVAL);
2855 }
2856
2857 static inline void
2858 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2859 {
2860         struct thread *t = &p->threads[p->thread_id];
2861         struct instruction *ip = t->ip;
2862         uint32_t done;
2863
2864         /* Extern object member function execute. */
2865         done = __instr_extern_obj_exec(p, t, ip);
2866
2867         /* Thread. */
2868         thread_ip_inc_cond(t, done);
2869         thread_yield_cond(p, done ^ 1);
2870 }
2871
2872 static inline void
2873 instr_extern_func_exec(struct rte_swx_pipeline *p)
2874 {
2875         struct thread *t = &p->threads[p->thread_id];
2876         struct instruction *ip = t->ip;
2877         uint32_t done;
2878
2879         /* Extern function execute. */
2880         done = __instr_extern_func_exec(p, t, ip);
2881
2882         /* Thread. */
2883         thread_ip_inc_cond(t, done);
2884         thread_yield_cond(p, done ^ 1);
2885 }
2886
2887 /*
2888  * hash.
2889  */
2890 static int
2891 instr_hash_translate(struct rte_swx_pipeline *p,
2892                      struct action *action,
2893                      char **tokens,
2894                      int n_tokens,
2895                      struct instruction *instr,
2896                      struct instruction_data *data __rte_unused)
2897 {
2898         struct hash_func *func;
2899         struct field *dst, *src_first, *src_last;
2900         uint32_t src_struct_id_first = 0, src_struct_id_last = 0;
2901
2902         CHECK(n_tokens == 5, EINVAL);
2903
2904         func = hash_func_find(p, tokens[1]);
2905         CHECK(func, EINVAL);
2906
2907         dst = metadata_field_parse(p, tokens[2]);
2908         CHECK(dst, EINVAL);
2909
2910         src_first = struct_field_parse(p, action, tokens[3], &src_struct_id_first);
2911         CHECK(src_first, EINVAL);
2912
2913         src_last = struct_field_parse(p, action, tokens[4], &src_struct_id_last);
2914         CHECK(src_last, EINVAL);
2915         CHECK(src_struct_id_first == src_struct_id_last, EINVAL);
2916
2917         instr->type = INSTR_HASH_FUNC;
2918         instr->hash_func.hash_func_id = (uint8_t)func->id;
2919         instr->hash_func.dst.offset = (uint8_t)dst->offset / 8;
2920         instr->hash_func.dst.n_bits = (uint8_t)dst->n_bits;
2921         instr->hash_func.src.struct_id = (uint8_t)src_struct_id_first;
2922         instr->hash_func.src.offset = (uint16_t)src_first->offset / 8;
2923         instr->hash_func.src.n_bytes = (uint16_t)((src_last->offset + src_last->n_bits -
2924                 src_first->offset) / 8);
2925
2926         return 0;
2927 }
2928
2929 static inline void
2930 instr_hash_func_exec(struct rte_swx_pipeline *p)
2931 {
2932         struct thread *t = &p->threads[p->thread_id];
2933         struct instruction *ip = t->ip;
2934
2935         /* Extern function execute. */
2936         __instr_hash_func_exec(p, t, ip);
2937
2938         /* Thread. */
2939         thread_ip_inc(p);
2940 }
2941
2942 /*
2943  * mov.
2944  */
2945 static int
2946 instr_mov_translate(struct rte_swx_pipeline *p,
2947                     struct action *action,
2948                     char **tokens,
2949                     int n_tokens,
2950                     struct instruction *instr,
2951                     struct instruction_data *data __rte_unused)
2952 {
2953         char *dst = tokens[1], *src = tokens[2];
2954         struct field *fdst, *fsrc;
2955         uint64_t src_val;
2956         uint32_t dst_struct_id = 0, src_struct_id = 0;
2957
2958         CHECK(n_tokens == 3, EINVAL);
2959
2960         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2961         CHECK(fdst, EINVAL);
2962         CHECK(!fdst->var_size, EINVAL);
2963
2964         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2965         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2966         if (fsrc) {
2967                 CHECK(!fsrc->var_size, EINVAL);
2968
2969                 instr->type = INSTR_MOV;
2970                 if (dst[0] != 'h' && src[0] == 'h')
2971                         instr->type = INSTR_MOV_MH;
2972                 if (dst[0] == 'h' && src[0] != 'h')
2973                         instr->type = INSTR_MOV_HM;
2974                 if (dst[0] == 'h' && src[0] == 'h')
2975                         instr->type = INSTR_MOV_HH;
2976
2977                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2978                 instr->mov.dst.n_bits = fdst->n_bits;
2979                 instr->mov.dst.offset = fdst->offset / 8;
2980                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2981                 instr->mov.src.n_bits = fsrc->n_bits;
2982                 instr->mov.src.offset = fsrc->offset / 8;
2983                 return 0;
2984         }
2985
2986         /* MOV_I. */
2987         src_val = strtoull(src, &src, 0);
2988         CHECK(!src[0], EINVAL);
2989
2990         if (dst[0] == 'h')
2991                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2992
2993         instr->type = INSTR_MOV_I;
2994         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2995         instr->mov.dst.n_bits = fdst->n_bits;
2996         instr->mov.dst.offset = fdst->offset / 8;
2997         instr->mov.src_val = src_val;
2998         return 0;
2999 }
3000
3001 static inline void
3002 instr_mov_exec(struct rte_swx_pipeline *p)
3003 {
3004         struct thread *t = &p->threads[p->thread_id];
3005         struct instruction *ip = t->ip;
3006
3007         __instr_mov_exec(p, t, ip);
3008
3009         /* Thread. */
3010         thread_ip_inc(p);
3011 }
3012
3013 static inline void
3014 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3015 {
3016         struct thread *t = &p->threads[p->thread_id];
3017         struct instruction *ip = t->ip;
3018
3019         __instr_mov_mh_exec(p, t, ip);
3020
3021         /* Thread. */
3022         thread_ip_inc(p);
3023 }
3024
3025 static inline void
3026 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3027 {
3028         struct thread *t = &p->threads[p->thread_id];
3029         struct instruction *ip = t->ip;
3030
3031         __instr_mov_hm_exec(p, t, ip);
3032
3033         /* Thread. */
3034         thread_ip_inc(p);
3035 }
3036
3037 static inline void
3038 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3039 {
3040         struct thread *t = &p->threads[p->thread_id];
3041         struct instruction *ip = t->ip;
3042
3043         __instr_mov_hh_exec(p, t, ip);
3044
3045         /* Thread. */
3046         thread_ip_inc(p);
3047 }
3048
3049 static inline void
3050 instr_mov_i_exec(struct rte_swx_pipeline *p)
3051 {
3052         struct thread *t = &p->threads[p->thread_id];
3053         struct instruction *ip = t->ip;
3054
3055         __instr_mov_i_exec(p, t, ip);
3056
3057         /* Thread. */
3058         thread_ip_inc(p);
3059 }
3060
3061 /*
3062  * dma.
3063  */
3064 static inline void
3065 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3066 {
3067         struct thread *t = &p->threads[p->thread_id];
3068         struct instruction *ip = t->ip;
3069
3070         __instr_dma_ht_exec(p, t, ip);
3071
3072         /* Thread. */
3073         thread_ip_inc(p);
3074 }
3075
3076 static inline void
3077 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3078 {
3079         struct thread *t = &p->threads[p->thread_id];
3080         struct instruction *ip = t->ip;
3081
3082         __instr_dma_ht2_exec(p, t, ip);
3083
3084         /* Thread. */
3085         thread_ip_inc(p);
3086 }
3087
3088 static inline void
3089 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3090 {
3091         struct thread *t = &p->threads[p->thread_id];
3092         struct instruction *ip = t->ip;
3093
3094         __instr_dma_ht3_exec(p, t, ip);
3095
3096         /* Thread. */
3097         thread_ip_inc(p);
3098 }
3099
3100 static inline void
3101 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3102 {
3103         struct thread *t = &p->threads[p->thread_id];
3104         struct instruction *ip = t->ip;
3105
3106         __instr_dma_ht4_exec(p, t, ip);
3107
3108         /* Thread. */
3109         thread_ip_inc(p);
3110 }
3111
3112 static inline void
3113 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3114 {
3115         struct thread *t = &p->threads[p->thread_id];
3116         struct instruction *ip = t->ip;
3117
3118         __instr_dma_ht5_exec(p, t, ip);
3119
3120         /* Thread. */
3121         thread_ip_inc(p);
3122 }
3123
3124 static inline void
3125 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3126 {
3127         struct thread *t = &p->threads[p->thread_id];
3128         struct instruction *ip = t->ip;
3129
3130         __instr_dma_ht6_exec(p, t, ip);
3131
3132         /* Thread. */
3133         thread_ip_inc(p);
3134 }
3135
3136 static inline void
3137 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3138 {
3139         struct thread *t = &p->threads[p->thread_id];
3140         struct instruction *ip = t->ip;
3141
3142         __instr_dma_ht7_exec(p, t, ip);
3143
3144         /* Thread. */
3145         thread_ip_inc(p);
3146 }
3147
3148 static inline void
3149 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3150 {
3151         struct thread *t = &p->threads[p->thread_id];
3152         struct instruction *ip = t->ip;
3153
3154         __instr_dma_ht8_exec(p, t, ip);
3155
3156         /* Thread. */
3157         thread_ip_inc(p);
3158 }
3159
3160 /*
3161  * alu.
3162  */
3163 static int
3164 instr_alu_add_translate(struct rte_swx_pipeline *p,
3165                         struct action *action,
3166                         char **tokens,
3167                         int n_tokens,
3168                         struct instruction *instr,
3169                         struct instruction_data *data __rte_unused)
3170 {
3171         char *dst = tokens[1], *src = tokens[2];
3172         struct field *fdst, *fsrc;
3173         uint64_t src_val;
3174         uint32_t dst_struct_id = 0, src_struct_id = 0;
3175
3176         CHECK(n_tokens == 3, EINVAL);
3177
3178         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3179         CHECK(fdst, EINVAL);
3180         CHECK(!fdst->var_size, EINVAL);
3181
3182         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3183         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3184         if (fsrc) {
3185                 CHECK(!fsrc->var_size, EINVAL);
3186
3187                 instr->type = INSTR_ALU_ADD;
3188                 if (dst[0] == 'h' && src[0] != 'h')
3189                         instr->type = INSTR_ALU_ADD_HM;
3190                 if (dst[0] != 'h' && src[0] == 'h')
3191                         instr->type = INSTR_ALU_ADD_MH;
3192                 if (dst[0] == 'h' && src[0] == 'h')
3193                         instr->type = INSTR_ALU_ADD_HH;
3194
3195                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3196                 instr->alu.dst.n_bits = fdst->n_bits;
3197                 instr->alu.dst.offset = fdst->offset / 8;
3198                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3199                 instr->alu.src.n_bits = fsrc->n_bits;
3200                 instr->alu.src.offset = fsrc->offset / 8;
3201                 return 0;
3202         }
3203
3204         /* ADD_MI, ADD_HI. */
3205         src_val = strtoull(src, &src, 0);
3206         CHECK(!src[0], EINVAL);
3207
3208         instr->type = INSTR_ALU_ADD_MI;
3209         if (dst[0] == 'h')
3210                 instr->type = INSTR_ALU_ADD_HI;
3211
3212         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3213         instr->alu.dst.n_bits = fdst->n_bits;
3214         instr->alu.dst.offset = fdst->offset / 8;
3215         instr->alu.src_val = src_val;
3216         return 0;
3217 }
3218
3219 static int
3220 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3221                         struct action *action,
3222                         char **tokens,
3223                         int n_tokens,
3224                         struct instruction *instr,
3225                         struct instruction_data *data __rte_unused)
3226 {
3227         char *dst = tokens[1], *src = tokens[2];
3228         struct field *fdst, *fsrc;
3229         uint64_t src_val;
3230         uint32_t dst_struct_id = 0, src_struct_id = 0;
3231
3232         CHECK(n_tokens == 3, EINVAL);
3233
3234         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3235         CHECK(fdst, EINVAL);
3236         CHECK(!fdst->var_size, EINVAL);
3237
3238         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3239         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3240         if (fsrc) {
3241                 CHECK(!fsrc->var_size, EINVAL);
3242
3243                 instr->type = INSTR_ALU_SUB;
3244                 if (dst[0] == 'h' && src[0] != 'h')
3245                         instr->type = INSTR_ALU_SUB_HM;
3246                 if (dst[0] != 'h' && src[0] == 'h')
3247                         instr->type = INSTR_ALU_SUB_MH;
3248                 if (dst[0] == 'h' && src[0] == 'h')
3249                         instr->type = INSTR_ALU_SUB_HH;
3250
3251                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3252                 instr->alu.dst.n_bits = fdst->n_bits;
3253                 instr->alu.dst.offset = fdst->offset / 8;
3254                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3255                 instr->alu.src.n_bits = fsrc->n_bits;
3256                 instr->alu.src.offset = fsrc->offset / 8;
3257                 return 0;
3258         }
3259
3260         /* SUB_MI, SUB_HI. */
3261         src_val = strtoull(src, &src, 0);
3262         CHECK(!src[0], EINVAL);
3263
3264         instr->type = INSTR_ALU_SUB_MI;
3265         if (dst[0] == 'h')
3266                 instr->type = INSTR_ALU_SUB_HI;
3267
3268         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3269         instr->alu.dst.n_bits = fdst->n_bits;
3270         instr->alu.dst.offset = fdst->offset / 8;
3271         instr->alu.src_val = src_val;
3272         return 0;
3273 }
3274
3275 static int
3276 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3277                           struct action *action __rte_unused,
3278                           char **tokens,
3279                           int n_tokens,
3280                           struct instruction *instr,
3281                           struct instruction_data *data __rte_unused)
3282 {
3283         char *dst = tokens[1], *src = tokens[2];
3284         struct header *hdst, *hsrc;
3285         struct field *fdst, *fsrc;
3286
3287         CHECK(n_tokens == 3, EINVAL);
3288
3289         fdst = header_field_parse(p, dst, &hdst);
3290         CHECK(fdst, EINVAL);
3291         CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3292
3293         /* CKADD_FIELD. */
3294         fsrc = header_field_parse(p, src, &hsrc);
3295         if (fsrc) {
3296                 CHECK(!fsrc->var_size, EINVAL);
3297
3298                 instr->type = INSTR_ALU_CKADD_FIELD;
3299                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3300                 instr->alu.dst.n_bits = fdst->n_bits;
3301                 instr->alu.dst.offset = fdst->offset / 8;
3302                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3303                 instr->alu.src.n_bits = fsrc->n_bits;
3304                 instr->alu.src.offset = fsrc->offset / 8;
3305                 return 0;
3306         }
3307
3308         /* CKADD_STRUCT, CKADD_STRUCT20. */
3309         hsrc = header_parse(p, src);
3310         CHECK(hsrc, EINVAL);
3311
3312         instr->type = INSTR_ALU_CKADD_STRUCT;
3313         if (!hsrc->st->var_size && ((hsrc->st->n_bits / 8) == 20))
3314                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3315
3316         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3317         instr->alu.dst.n_bits = fdst->n_bits;
3318         instr->alu.dst.offset = fdst->offset / 8;
3319         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3320         instr->alu.src.n_bits = (uint8_t)hsrc->id; /* The src header ID is stored here. */
3321         instr->alu.src.offset = 0; /* Unused. */
3322         return 0;
3323 }
3324
3325 static int
3326 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3327                           struct action *action __rte_unused,
3328                           char **tokens,
3329                           int n_tokens,
3330                           struct instruction *instr,
3331                           struct instruction_data *data __rte_unused)
3332 {
3333         char *dst = tokens[1], *src = tokens[2];
3334         struct header *hdst, *hsrc;
3335         struct field *fdst, *fsrc;
3336
3337         CHECK(n_tokens == 3, EINVAL);
3338
3339         fdst = header_field_parse(p, dst, &hdst);
3340         CHECK(fdst, EINVAL);
3341         CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3342
3343         fsrc = header_field_parse(p, src, &hsrc);
3344         CHECK(fsrc, EINVAL);
3345         CHECK(!fsrc->var_size, EINVAL);
3346
3347         instr->type = INSTR_ALU_CKSUB_FIELD;
3348         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3349         instr->alu.dst.n_bits = fdst->n_bits;
3350         instr->alu.dst.offset = fdst->offset / 8;
3351         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3352         instr->alu.src.n_bits = fsrc->n_bits;
3353         instr->alu.src.offset = fsrc->offset / 8;
3354         return 0;
3355 }
3356
3357 static int
3358 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3359                         struct action *action,
3360                         char **tokens,
3361                         int n_tokens,
3362                         struct instruction *instr,
3363                         struct instruction_data *data __rte_unused)
3364 {
3365         char *dst = tokens[1], *src = tokens[2];
3366         struct field *fdst, *fsrc;
3367         uint64_t src_val;
3368         uint32_t dst_struct_id = 0, src_struct_id = 0;
3369
3370         CHECK(n_tokens == 3, EINVAL);
3371
3372         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3373         CHECK(fdst, EINVAL);
3374         CHECK(!fdst->var_size, EINVAL);
3375
3376         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3377         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3378         if (fsrc) {
3379                 CHECK(!fsrc->var_size, EINVAL);
3380
3381                 instr->type = INSTR_ALU_SHL;
3382                 if (dst[0] == 'h' && src[0] != 'h')
3383                         instr->type = INSTR_ALU_SHL_HM;
3384                 if (dst[0] != 'h' && src[0] == 'h')
3385                         instr->type = INSTR_ALU_SHL_MH;
3386                 if (dst[0] == 'h' && src[0] == 'h')
3387                         instr->type = INSTR_ALU_SHL_HH;
3388
3389                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3390                 instr->alu.dst.n_bits = fdst->n_bits;
3391                 instr->alu.dst.offset = fdst->offset / 8;
3392                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3393                 instr->alu.src.n_bits = fsrc->n_bits;
3394                 instr->alu.src.offset = fsrc->offset / 8;
3395                 return 0;
3396         }
3397
3398         /* SHL_MI, SHL_HI. */
3399         src_val = strtoull(src, &src, 0);
3400         CHECK(!src[0], EINVAL);
3401
3402         instr->type = INSTR_ALU_SHL_MI;
3403         if (dst[0] == 'h')
3404                 instr->type = INSTR_ALU_SHL_HI;
3405
3406         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3407         instr->alu.dst.n_bits = fdst->n_bits;
3408         instr->alu.dst.offset = fdst->offset / 8;
3409         instr->alu.src_val = src_val;
3410         return 0;
3411 }
3412
3413 static int
3414 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3415                         struct action *action,
3416                         char **tokens,
3417                         int n_tokens,
3418                         struct instruction *instr,
3419                         struct instruction_data *data __rte_unused)
3420 {
3421         char *dst = tokens[1], *src = tokens[2];
3422         struct field *fdst, *fsrc;
3423         uint64_t src_val;
3424         uint32_t dst_struct_id = 0, src_struct_id = 0;
3425
3426         CHECK(n_tokens == 3, EINVAL);
3427
3428         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3429         CHECK(fdst, EINVAL);
3430         CHECK(!fdst->var_size, EINVAL);
3431
3432         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3433         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3434         if (fsrc) {
3435                 CHECK(!fsrc->var_size, EINVAL);
3436
3437                 instr->type = INSTR_ALU_SHR;
3438                 if (dst[0] == 'h' && src[0] != 'h')
3439                         instr->type = INSTR_ALU_SHR_HM;
3440                 if (dst[0] != 'h' && src[0] == 'h')
3441                         instr->type = INSTR_ALU_SHR_MH;
3442                 if (dst[0] == 'h' && src[0] == 'h')
3443                         instr->type = INSTR_ALU_SHR_HH;
3444
3445                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3446                 instr->alu.dst.n_bits = fdst->n_bits;
3447                 instr->alu.dst.offset = fdst->offset / 8;
3448                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3449                 instr->alu.src.n_bits = fsrc->n_bits;
3450                 instr->alu.src.offset = fsrc->offset / 8;
3451                 return 0;
3452         }
3453
3454         /* SHR_MI, SHR_HI. */
3455         src_val = strtoull(src, &src, 0);
3456         CHECK(!src[0], EINVAL);
3457
3458         instr->type = INSTR_ALU_SHR_MI;
3459         if (dst[0] == 'h')
3460                 instr->type = INSTR_ALU_SHR_HI;
3461
3462         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3463         instr->alu.dst.n_bits = fdst->n_bits;
3464         instr->alu.dst.offset = fdst->offset / 8;
3465         instr->alu.src_val = src_val;
3466         return 0;
3467 }
3468
3469 static int
3470 instr_alu_and_translate(struct rte_swx_pipeline *p,
3471                         struct action *action,
3472                         char **tokens,
3473                         int n_tokens,
3474                         struct instruction *instr,
3475                         struct instruction_data *data __rte_unused)
3476 {
3477         char *dst = tokens[1], *src = tokens[2];
3478         struct field *fdst, *fsrc;
3479         uint64_t src_val;
3480         uint32_t dst_struct_id = 0, src_struct_id = 0;
3481
3482         CHECK(n_tokens == 3, EINVAL);
3483
3484         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3485         CHECK(fdst, EINVAL);
3486         CHECK(!fdst->var_size, EINVAL);
3487
3488         /* AND, AND_MH, AND_HM, AND_HH. */
3489         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3490         if (fsrc) {
3491                 CHECK(!fsrc->var_size, EINVAL);
3492
3493                 instr->type = INSTR_ALU_AND;
3494                 if (dst[0] != 'h' && src[0] == 'h')
3495                         instr->type = INSTR_ALU_AND_MH;
3496                 if (dst[0] == 'h' && src[0] != 'h')
3497                         instr->type = INSTR_ALU_AND_HM;
3498                 if (dst[0] == 'h' && src[0] == 'h')
3499                         instr->type = INSTR_ALU_AND_HH;
3500
3501                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3502                 instr->alu.dst.n_bits = fdst->n_bits;
3503                 instr->alu.dst.offset = fdst->offset / 8;
3504                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3505                 instr->alu.src.n_bits = fsrc->n_bits;
3506                 instr->alu.src.offset = fsrc->offset / 8;
3507                 return 0;
3508         }
3509
3510         /* AND_I. */
3511         src_val = strtoull(src, &src, 0);
3512         CHECK(!src[0], EINVAL);
3513
3514         if (dst[0] == 'h')
3515                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3516
3517         instr->type = INSTR_ALU_AND_I;
3518         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3519         instr->alu.dst.n_bits = fdst->n_bits;
3520         instr->alu.dst.offset = fdst->offset / 8;
3521         instr->alu.src_val = src_val;
3522         return 0;
3523 }
3524
3525 static int
3526 instr_alu_or_translate(struct rte_swx_pipeline *p,
3527                        struct action *action,
3528                        char **tokens,
3529                        int n_tokens,
3530                        struct instruction *instr,
3531                        struct instruction_data *data __rte_unused)
3532 {
3533         char *dst = tokens[1], *src = tokens[2];
3534         struct field *fdst, *fsrc;
3535         uint64_t src_val;
3536         uint32_t dst_struct_id = 0, src_struct_id = 0;
3537
3538         CHECK(n_tokens == 3, EINVAL);
3539
3540         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3541         CHECK(fdst, EINVAL);
3542         CHECK(!fdst->var_size, EINVAL);
3543
3544         /* OR, OR_MH, OR_HM, OR_HH. */
3545         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3546         if (fsrc) {
3547                 CHECK(!fsrc->var_size, EINVAL);
3548
3549                 instr->type = INSTR_ALU_OR;
3550                 if (dst[0] != 'h' && src[0] == 'h')
3551                         instr->type = INSTR_ALU_OR_MH;
3552                 if (dst[0] == 'h' && src[0] != 'h')
3553                         instr->type = INSTR_ALU_OR_HM;
3554                 if (dst[0] == 'h' && src[0] == 'h')
3555                         instr->type = INSTR_ALU_OR_HH;
3556
3557                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3558                 instr->alu.dst.n_bits = fdst->n_bits;
3559                 instr->alu.dst.offset = fdst->offset / 8;
3560                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3561                 instr->alu.src.n_bits = fsrc->n_bits;
3562                 instr->alu.src.offset = fsrc->offset / 8;
3563                 return 0;
3564         }
3565
3566         /* OR_I. */
3567         src_val = strtoull(src, &src, 0);
3568         CHECK(!src[0], EINVAL);
3569
3570         if (dst[0] == 'h')
3571                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3572
3573         instr->type = INSTR_ALU_OR_I;
3574         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3575         instr->alu.dst.n_bits = fdst->n_bits;
3576         instr->alu.dst.offset = fdst->offset / 8;
3577         instr->alu.src_val = src_val;
3578         return 0;
3579 }
3580
3581 static int
3582 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3583                         struct action *action,
3584                         char **tokens,
3585                         int n_tokens,
3586                         struct instruction *instr,
3587                         struct instruction_data *data __rte_unused)
3588 {
3589         char *dst = tokens[1], *src = tokens[2];
3590         struct field *fdst, *fsrc;
3591         uint64_t src_val;
3592         uint32_t dst_struct_id = 0, src_struct_id = 0;
3593
3594         CHECK(n_tokens == 3, EINVAL);
3595
3596         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3597         CHECK(fdst, EINVAL);
3598         CHECK(!fdst->var_size, EINVAL);
3599
3600         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3601         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3602         if (fsrc) {
3603                 CHECK(!fsrc->var_size, EINVAL);
3604
3605                 instr->type = INSTR_ALU_XOR;
3606                 if (dst[0] != 'h' && src[0] == 'h')
3607                         instr->type = INSTR_ALU_XOR_MH;
3608                 if (dst[0] == 'h' && src[0] != 'h')
3609                         instr->type = INSTR_ALU_XOR_HM;
3610                 if (dst[0] == 'h' && src[0] == 'h')
3611                         instr->type = INSTR_ALU_XOR_HH;
3612
3613                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3614                 instr->alu.dst.n_bits = fdst->n_bits;
3615                 instr->alu.dst.offset = fdst->offset / 8;
3616                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3617                 instr->alu.src.n_bits = fsrc->n_bits;
3618                 instr->alu.src.offset = fsrc->offset / 8;
3619                 return 0;
3620         }
3621
3622         /* XOR_I. */
3623         src_val = strtoull(src, &src, 0);
3624         CHECK(!src[0], EINVAL);
3625
3626         if (dst[0] == 'h')
3627                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3628
3629         instr->type = INSTR_ALU_XOR_I;
3630         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3631         instr->alu.dst.n_bits = fdst->n_bits;
3632         instr->alu.dst.offset = fdst->offset / 8;
3633         instr->alu.src_val = src_val;
3634         return 0;
3635 }
3636
3637 static inline void
3638 instr_alu_add_exec(struct rte_swx_pipeline *p)
3639 {
3640         struct thread *t = &p->threads[p->thread_id];
3641         struct instruction *ip = t->ip;
3642
3643         /* Structs */
3644         __instr_alu_add_exec(p, t, ip);
3645
3646         /* Thread. */
3647         thread_ip_inc(p);
3648 }
3649
3650 static inline void
3651 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3652 {
3653         struct thread *t = &p->threads[p->thread_id];
3654         struct instruction *ip = t->ip;
3655
3656         /* Structs. */
3657         __instr_alu_add_mh_exec(p, t, ip);
3658
3659         /* Thread. */
3660         thread_ip_inc(p);
3661 }
3662
3663 static inline void
3664 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3665 {
3666         struct thread *t = &p->threads[p->thread_id];
3667         struct instruction *ip = t->ip;
3668
3669         /* Structs. */
3670         __instr_alu_add_hm_exec(p, t, ip);
3671
3672         /* Thread. */
3673         thread_ip_inc(p);
3674 }
3675
3676 static inline void
3677 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3678 {
3679         struct thread *t = &p->threads[p->thread_id];
3680         struct instruction *ip = t->ip;
3681
3682         /* Structs. */
3683         __instr_alu_add_hh_exec(p, t, ip);
3684
3685         /* Thread. */
3686         thread_ip_inc(p);
3687 }
3688
3689 static inline void
3690 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3691 {
3692         struct thread *t = &p->threads[p->thread_id];
3693         struct instruction *ip = t->ip;
3694
3695         /* Structs. */
3696         __instr_alu_add_mi_exec(p, t, ip);
3697
3698         /* Thread. */
3699         thread_ip_inc(p);
3700 }
3701
3702 static inline void
3703 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3704 {
3705         struct thread *t = &p->threads[p->thread_id];
3706         struct instruction *ip = t->ip;
3707
3708         /* Structs. */
3709         __instr_alu_add_hi_exec(p, t, ip);
3710
3711         /* Thread. */
3712         thread_ip_inc(p);
3713 }
3714
3715 static inline void
3716 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3717 {
3718         struct thread *t = &p->threads[p->thread_id];
3719         struct instruction *ip = t->ip;
3720
3721         /* Structs. */
3722         __instr_alu_sub_exec(p, t, ip);
3723
3724         /* Thread. */
3725         thread_ip_inc(p);
3726 }
3727
3728 static inline void
3729 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3730 {
3731         struct thread *t = &p->threads[p->thread_id];
3732         struct instruction *ip = t->ip;
3733
3734         /* Structs. */
3735         __instr_alu_sub_mh_exec(p, t, ip);
3736
3737         /* Thread. */
3738         thread_ip_inc(p);
3739 }
3740
3741 static inline void
3742 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3743 {
3744         struct thread *t = &p->threads[p->thread_id];
3745         struct instruction *ip = t->ip;
3746
3747         /* Structs. */
3748         __instr_alu_sub_hm_exec(p, t, ip);
3749
3750         /* Thread. */
3751         thread_ip_inc(p);
3752 }
3753
3754 static inline void
3755 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3756 {
3757         struct thread *t = &p->threads[p->thread_id];
3758         struct instruction *ip = t->ip;
3759
3760         /* Structs. */
3761         __instr_alu_sub_hh_exec(p, t, ip);
3762
3763         /* Thread. */
3764         thread_ip_inc(p);
3765 }
3766
3767 static inline void
3768 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3769 {
3770         struct thread *t = &p->threads[p->thread_id];
3771         struct instruction *ip = t->ip;
3772
3773         /* Structs. */
3774         __instr_alu_sub_mi_exec(p, t, ip);
3775
3776         /* Thread. */
3777         thread_ip_inc(p);
3778 }
3779
3780 static inline void
3781 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3782 {
3783         struct thread *t = &p->threads[p->thread_id];
3784         struct instruction *ip = t->ip;
3785
3786         /* Structs. */
3787         __instr_alu_sub_hi_exec(p, t, ip);
3788
3789         /* Thread. */
3790         thread_ip_inc(p);
3791 }
3792
3793 static inline void
3794 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3795 {
3796         struct thread *t = &p->threads[p->thread_id];
3797         struct instruction *ip = t->ip;
3798
3799         /* Structs. */
3800         __instr_alu_shl_exec(p, t, ip);
3801
3802         /* Thread. */
3803         thread_ip_inc(p);
3804 }
3805
3806 static inline void
3807 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3808 {
3809         struct thread *t = &p->threads[p->thread_id];
3810         struct instruction *ip = t->ip;
3811
3812         /* Structs. */
3813         __instr_alu_shl_mh_exec(p, t, ip);
3814
3815         /* Thread. */
3816         thread_ip_inc(p);
3817 }
3818
3819 static inline void
3820 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3821 {
3822         struct thread *t = &p->threads[p->thread_id];
3823         struct instruction *ip = t->ip;
3824
3825         /* Structs. */
3826         __instr_alu_shl_hm_exec(p, t, ip);
3827
3828         /* Thread. */
3829         thread_ip_inc(p);
3830 }
3831
3832 static inline void
3833 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3834 {
3835         struct thread *t = &p->threads[p->thread_id];
3836         struct instruction *ip = t->ip;
3837
3838         /* Structs. */
3839         __instr_alu_shl_hh_exec(p, t, ip);
3840
3841         /* Thread. */
3842         thread_ip_inc(p);
3843 }
3844
3845 static inline void
3846 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3847 {
3848         struct thread *t = &p->threads[p->thread_id];
3849         struct instruction *ip = t->ip;
3850
3851         /* Structs. */
3852         __instr_alu_shl_mi_exec(p, t, ip);
3853
3854         /* Thread. */
3855         thread_ip_inc(p);
3856 }
3857
3858 static inline void
3859 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3860 {
3861         struct thread *t = &p->threads[p->thread_id];
3862         struct instruction *ip = t->ip;
3863
3864         /* Structs. */
3865         __instr_alu_shl_hi_exec(p, t, ip);
3866
3867         /* Thread. */
3868         thread_ip_inc(p);
3869 }
3870
3871 static inline void
3872 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3873 {
3874         struct thread *t = &p->threads[p->thread_id];
3875         struct instruction *ip = t->ip;
3876
3877         /* Structs. */
3878         __instr_alu_shr_exec(p, t, ip);
3879
3880         /* Thread. */
3881         thread_ip_inc(p);
3882 }
3883
3884 static inline void
3885 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3886 {
3887         struct thread *t = &p->threads[p->thread_id];
3888         struct instruction *ip = t->ip;
3889
3890         /* Structs. */
3891         __instr_alu_shr_mh_exec(p, t, ip);
3892
3893         /* Thread. */
3894         thread_ip_inc(p);
3895 }
3896
3897 static inline void
3898 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3899 {
3900         struct thread *t = &p->threads[p->thread_id];
3901         struct instruction *ip = t->ip;
3902
3903         /* Structs. */
3904         __instr_alu_shr_hm_exec(p, t, ip);
3905
3906         /* Thread. */
3907         thread_ip_inc(p);
3908 }
3909
3910 static inline void
3911 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3912 {
3913         struct thread *t = &p->threads[p->thread_id];
3914         struct instruction *ip = t->ip;
3915
3916         /* Structs. */
3917         __instr_alu_shr_hh_exec(p, t, ip);
3918
3919         /* Thread. */
3920         thread_ip_inc(p);
3921 }
3922
3923 static inline void
3924 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3925 {
3926         struct thread *t = &p->threads[p->thread_id];
3927         struct instruction *ip = t->ip;
3928
3929         /* Structs. */
3930         __instr_alu_shr_mi_exec(p, t, ip);
3931
3932         /* Thread. */
3933         thread_ip_inc(p);
3934 }
3935
3936 static inline void
3937 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3938 {
3939         struct thread *t = &p->threads[p->thread_id];
3940         struct instruction *ip = t->ip;
3941
3942         /* Structs. */
3943         __instr_alu_shr_hi_exec(p, t, ip);
3944
3945         /* Thread. */
3946         thread_ip_inc(p);
3947 }
3948
3949 static inline void
3950 instr_alu_and_exec(struct rte_swx_pipeline *p)
3951 {
3952         struct thread *t = &p->threads[p->thread_id];
3953         struct instruction *ip = t->ip;
3954
3955         /* Structs. */
3956         __instr_alu_and_exec(p, t, ip);
3957
3958         /* Thread. */
3959         thread_ip_inc(p);
3960 }
3961
3962 static inline void
3963 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3964 {
3965         struct thread *t = &p->threads[p->thread_id];
3966         struct instruction *ip = t->ip;
3967
3968         /* Structs. */
3969         __instr_alu_and_mh_exec(p, t, ip);
3970
3971         /* Thread. */
3972         thread_ip_inc(p);
3973 }
3974
3975 static inline void
3976 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3977 {
3978         struct thread *t = &p->threads[p->thread_id];
3979         struct instruction *ip = t->ip;
3980
3981         /* Structs. */
3982         __instr_alu_and_hm_exec(p, t, ip);
3983
3984         /* Thread. */
3985         thread_ip_inc(p);
3986 }
3987
3988 static inline void
3989 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3990 {
3991         struct thread *t = &p->threads[p->thread_id];
3992         struct instruction *ip = t->ip;
3993
3994         /* Structs. */
3995         __instr_alu_and_hh_exec(p, t, ip);
3996
3997         /* Thread. */
3998         thread_ip_inc(p);
3999 }
4000
4001 static inline void
4002 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4003 {
4004         struct thread *t = &p->threads[p->thread_id];
4005         struct instruction *ip = t->ip;
4006
4007         /* Structs. */
4008         __instr_alu_and_i_exec(p, t, ip);
4009
4010         /* Thread. */
4011         thread_ip_inc(p);
4012 }
4013
4014 static inline void
4015 instr_alu_or_exec(struct rte_swx_pipeline *p)
4016 {
4017         struct thread *t = &p->threads[p->thread_id];
4018         struct instruction *ip = t->ip;
4019
4020         /* Structs. */
4021         __instr_alu_or_exec(p, t, ip);
4022
4023         /* Thread. */
4024         thread_ip_inc(p);
4025 }
4026
4027 static inline void
4028 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
4029 {
4030         struct thread *t = &p->threads[p->thread_id];
4031         struct instruction *ip = t->ip;
4032
4033         /* Structs. */
4034         __instr_alu_or_mh_exec(p, t, ip);
4035
4036         /* Thread. */
4037         thread_ip_inc(p);
4038 }
4039
4040 static inline void
4041 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
4042 {
4043         struct thread *t = &p->threads[p->thread_id];
4044         struct instruction *ip = t->ip;
4045
4046         /* Structs. */
4047         __instr_alu_or_hm_exec(p, t, ip);
4048
4049         /* Thread. */
4050         thread_ip_inc(p);
4051 }
4052
4053 static inline void
4054 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
4055 {
4056         struct thread *t = &p->threads[p->thread_id];
4057         struct instruction *ip = t->ip;
4058
4059         /* Structs. */
4060         __instr_alu_or_hh_exec(p, t, ip);
4061
4062         /* Thread. */
4063         thread_ip_inc(p);
4064 }
4065
4066 static inline void
4067 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4068 {
4069         struct thread *t = &p->threads[p->thread_id];
4070         struct instruction *ip = t->ip;
4071
4072         /* Structs. */
4073         __instr_alu_or_i_exec(p, t, ip);
4074
4075         /* Thread. */
4076         thread_ip_inc(p);
4077 }
4078
4079 static inline void
4080 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4081 {
4082         struct thread *t = &p->threads[p->thread_id];
4083         struct instruction *ip = t->ip;
4084
4085         /* Structs. */
4086         __instr_alu_xor_exec(p, t, ip);
4087
4088         /* Thread. */
4089         thread_ip_inc(p);
4090 }
4091
4092 static inline void
4093 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
4094 {
4095         struct thread *t = &p->threads[p->thread_id];
4096         struct instruction *ip = t->ip;
4097
4098         /* Structs. */
4099         __instr_alu_xor_mh_exec(p, t, ip);
4100
4101         /* Thread. */
4102         thread_ip_inc(p);
4103 }
4104
4105 static inline void
4106 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
4107 {
4108         struct thread *t = &p->threads[p->thread_id];
4109         struct instruction *ip = t->ip;
4110
4111         /* Structs. */
4112         __instr_alu_xor_hm_exec(p, t, ip);
4113
4114         /* Thread. */
4115         thread_ip_inc(p);
4116 }
4117
4118 static inline void
4119 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
4120 {
4121         struct thread *t = &p->threads[p->thread_id];
4122         struct instruction *ip = t->ip;
4123
4124         /* Structs. */
4125         __instr_alu_xor_hh_exec(p, t, ip);
4126
4127         /* Thread. */
4128         thread_ip_inc(p);
4129 }
4130
4131 static inline void
4132 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4133 {
4134         struct thread *t = &p->threads[p->thread_id];
4135         struct instruction *ip = t->ip;
4136
4137         /* Structs. */
4138         __instr_alu_xor_i_exec(p, t, ip);
4139
4140         /* Thread. */
4141         thread_ip_inc(p);
4142 }
4143
4144 static inline void
4145 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4146 {
4147         struct thread *t = &p->threads[p->thread_id];
4148         struct instruction *ip = t->ip;
4149
4150         /* Structs. */
4151         __instr_alu_ckadd_field_exec(p, t, ip);
4152
4153         /* Thread. */
4154         thread_ip_inc(p);
4155 }
4156
4157 static inline void
4158 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4159 {
4160         struct thread *t = &p->threads[p->thread_id];
4161         struct instruction *ip = t->ip;
4162
4163         /* Structs. */
4164         __instr_alu_cksub_field_exec(p, t, ip);
4165
4166         /* Thread. */
4167         thread_ip_inc(p);
4168 }
4169
4170 static inline void
4171 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4172 {
4173         struct thread *t = &p->threads[p->thread_id];
4174         struct instruction *ip = t->ip;
4175
4176         /* Structs. */
4177         __instr_alu_ckadd_struct20_exec(p, t, ip);
4178
4179         /* Thread. */
4180         thread_ip_inc(p);
4181 }
4182
4183 static inline void
4184 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4185 {
4186         struct thread *t = &p->threads[p->thread_id];
4187         struct instruction *ip = t->ip;
4188
4189         /* Structs. */
4190         __instr_alu_ckadd_struct_exec(p, t, ip);
4191
4192         /* Thread. */
4193         thread_ip_inc(p);
4194 }
4195
4196 /*
4197  * Register array.
4198  */
4199 static struct regarray *
4200 regarray_find(struct rte_swx_pipeline *p, const char *name);
4201
4202 static int
4203 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4204                       struct action *action,
4205                       char **tokens,
4206                       int n_tokens,
4207                       struct instruction *instr,
4208                       struct instruction_data *data __rte_unused)
4209 {
4210         char *regarray = tokens[1], *idx = tokens[2];
4211         struct regarray *r;
4212         struct field *fidx;
4213         uint32_t idx_struct_id, idx_val;
4214
4215         CHECK(n_tokens == 3, EINVAL);
4216
4217         r = regarray_find(p, regarray);
4218         CHECK(r, EINVAL);
4219
4220         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4221         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4222         if (fidx) {
4223                 CHECK(!fidx->var_size, EINVAL);
4224
4225                 instr->type = INSTR_REGPREFETCH_RM;
4226                 if (idx[0] == 'h')
4227                         instr->type = INSTR_REGPREFETCH_RH;
4228
4229                 instr->regarray.regarray_id = r->id;
4230                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4231                 instr->regarray.idx.n_bits = fidx->n_bits;
4232                 instr->regarray.idx.offset = fidx->offset / 8;
4233                 instr->regarray.dstsrc_val = 0; /* Unused. */
4234                 return 0;
4235         }
4236
4237         /* REGPREFETCH_RI. */
4238         idx_val = strtoul(idx, &idx, 0);
4239         CHECK(!idx[0], EINVAL);
4240
4241         instr->type = INSTR_REGPREFETCH_RI;
4242         instr->regarray.regarray_id = r->id;
4243         instr->regarray.idx_val = idx_val;
4244         instr->regarray.dstsrc_val = 0; /* Unused. */
4245         return 0;
4246 }
4247
4248 static int
4249 instr_regrd_translate(struct rte_swx_pipeline *p,
4250                       struct action *action,
4251                       char **tokens,
4252                       int n_tokens,
4253                       struct instruction *instr,
4254                       struct instruction_data *data __rte_unused)
4255 {
4256         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4257         struct regarray *r;
4258         struct field *fdst, *fidx;
4259         uint32_t dst_struct_id, idx_struct_id, idx_val;
4260
4261         CHECK(n_tokens == 4, EINVAL);
4262
4263         r = regarray_find(p, regarray);
4264         CHECK(r, EINVAL);
4265
4266         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4267         CHECK(fdst, EINVAL);
4268         CHECK(!fdst->var_size, EINVAL);
4269
4270         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4271         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4272         if (fidx) {
4273                 CHECK(!fidx->var_size, EINVAL);
4274
4275                 instr->type = INSTR_REGRD_MRM;
4276                 if (dst[0] == 'h' && idx[0] != 'h')
4277                         instr->type = INSTR_REGRD_HRM;
4278                 if (dst[0] != 'h' && idx[0] == 'h')
4279                         instr->type = INSTR_REGRD_MRH;
4280                 if (dst[0] == 'h' && idx[0] == 'h')
4281                         instr->type = INSTR_REGRD_HRH;
4282
4283                 instr->regarray.regarray_id = r->id;
4284                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4285                 instr->regarray.idx.n_bits = fidx->n_bits;
4286                 instr->regarray.idx.offset = fidx->offset / 8;
4287                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4288                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4289                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4290                 return 0;
4291         }
4292
4293         /* REGRD_MRI, REGRD_HRI. */
4294         idx_val = strtoul(idx, &idx, 0);
4295         CHECK(!idx[0], EINVAL);
4296
4297         instr->type = INSTR_REGRD_MRI;
4298         if (dst[0] == 'h')
4299                 instr->type = INSTR_REGRD_HRI;
4300
4301         instr->regarray.regarray_id = r->id;
4302         instr->regarray.idx_val = idx_val;
4303         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4304         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4305         instr->regarray.dstsrc.offset = fdst->offset / 8;
4306         return 0;
4307 }
4308
4309 static int
4310 instr_regwr_translate(struct rte_swx_pipeline *p,
4311                       struct action *action,
4312                       char **tokens,
4313                       int n_tokens,
4314                       struct instruction *instr,
4315                       struct instruction_data *data __rte_unused)
4316 {
4317         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4318         struct regarray *r;
4319         struct field *fidx, *fsrc;
4320         uint64_t src_val;
4321         uint32_t idx_struct_id, idx_val, src_struct_id;
4322
4323         CHECK(n_tokens == 4, EINVAL);
4324
4325         r = regarray_find(p, regarray);
4326         CHECK(r, EINVAL);
4327
4328         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4329         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4330         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4331         if (fidx && fsrc) {
4332                 CHECK(!fidx->var_size, EINVAL);
4333                 CHECK(!fsrc->var_size, EINVAL);
4334
4335                 instr->type = INSTR_REGWR_RMM;
4336                 if (idx[0] == 'h' && src[0] != 'h')
4337                         instr->type = INSTR_REGWR_RHM;
4338                 if (idx[0] != 'h' && src[0] == 'h')
4339                         instr->type = INSTR_REGWR_RMH;
4340                 if (idx[0] == 'h' && src[0] == 'h')
4341                         instr->type = INSTR_REGWR_RHH;
4342
4343                 instr->regarray.regarray_id = r->id;
4344                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4345                 instr->regarray.idx.n_bits = fidx->n_bits;
4346                 instr->regarray.idx.offset = fidx->offset / 8;
4347                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4348                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4349                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4350                 return 0;
4351         }
4352
4353         /* REGWR_RHI, REGWR_RMI. */
4354         if (fidx && !fsrc) {
4355                 CHECK(!fidx->var_size, EINVAL);
4356
4357                 src_val = strtoull(src, &src, 0);
4358                 CHECK(!src[0], EINVAL);
4359
4360                 instr->type = INSTR_REGWR_RMI;
4361                 if (idx[0] == 'h')
4362                         instr->type = INSTR_REGWR_RHI;
4363
4364                 instr->regarray.regarray_id = r->id;
4365                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4366                 instr->regarray.idx.n_bits = fidx->n_bits;
4367                 instr->regarray.idx.offset = fidx->offset / 8;
4368                 instr->regarray.dstsrc_val = src_val;
4369                 return 0;
4370         }
4371
4372         /* REGWR_RIH, REGWR_RIM. */
4373         if (!fidx && fsrc) {
4374                 idx_val = strtoul(idx, &idx, 0);
4375                 CHECK(!idx[0], EINVAL);
4376
4377                 CHECK(!fsrc->var_size, EINVAL);
4378
4379                 instr->type = INSTR_REGWR_RIM;
4380                 if (src[0] == 'h')
4381                         instr->type = INSTR_REGWR_RIH;
4382
4383                 instr->regarray.regarray_id = r->id;
4384                 instr->regarray.idx_val = idx_val;
4385                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4386                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4387                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4388                 return 0;
4389         }
4390
4391         /* REGWR_RII. */
4392         src_val = strtoull(src, &src, 0);
4393         CHECK(!src[0], EINVAL);
4394
4395         idx_val = strtoul(idx, &idx, 0);
4396         CHECK(!idx[0], EINVAL);
4397
4398         instr->type = INSTR_REGWR_RII;
4399         instr->regarray.idx_val = idx_val;
4400         instr->regarray.dstsrc_val = src_val;
4401
4402         return 0;
4403 }
4404
4405 static int
4406 instr_regadd_translate(struct rte_swx_pipeline *p,
4407                        struct action *action,
4408                        char **tokens,
4409                        int n_tokens,
4410                        struct instruction *instr,
4411                        struct instruction_data *data __rte_unused)
4412 {
4413         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4414         struct regarray *r;
4415         struct field *fidx, *fsrc;
4416         uint64_t src_val;
4417         uint32_t idx_struct_id, idx_val, src_struct_id;
4418
4419         CHECK(n_tokens == 4, EINVAL);
4420
4421         r = regarray_find(p, regarray);
4422         CHECK(r, EINVAL);
4423
4424         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4425         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4426         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4427         if (fidx && fsrc) {
4428                 CHECK(!fidx->var_size, EINVAL);
4429                 CHECK(!fsrc->var_size, EINVAL);
4430
4431                 instr->type = INSTR_REGADD_RMM;
4432                 if (idx[0] == 'h' && src[0] != 'h')
4433                         instr->type = INSTR_REGADD_RHM;
4434                 if (idx[0] != 'h' && src[0] == 'h')
4435                         instr->type = INSTR_REGADD_RMH;
4436                 if (idx[0] == 'h' && src[0] == 'h')
4437                         instr->type = INSTR_REGADD_RHH;
4438
4439                 instr->regarray.regarray_id = r->id;
4440                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4441                 instr->regarray.idx.n_bits = fidx->n_bits;
4442                 instr->regarray.idx.offset = fidx->offset / 8;
4443                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4444                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4445                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4446                 return 0;
4447         }
4448
4449         /* REGADD_RHI, REGADD_RMI. */
4450         if (fidx && !fsrc) {
4451                 CHECK(!fidx->var_size, EINVAL);
4452
4453                 src_val = strtoull(src, &src, 0);
4454                 CHECK(!src[0], EINVAL);
4455
4456                 instr->type = INSTR_REGADD_RMI;
4457                 if (idx[0] == 'h')
4458                         instr->type = INSTR_REGADD_RHI;
4459
4460                 instr->regarray.regarray_id = r->id;
4461                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4462                 instr->regarray.idx.n_bits = fidx->n_bits;
4463                 instr->regarray.idx.offset = fidx->offset / 8;
4464                 instr->regarray.dstsrc_val = src_val;
4465                 return 0;
4466         }
4467
4468         /* REGADD_RIH, REGADD_RIM. */
4469         if (!fidx && fsrc) {
4470                 idx_val = strtoul(idx, &idx, 0);
4471                 CHECK(!idx[0], EINVAL);
4472
4473                 CHECK(!fsrc->var_size, EINVAL);
4474
4475                 instr->type = INSTR_REGADD_RIM;
4476                 if (src[0] == 'h')
4477                         instr->type = INSTR_REGADD_RIH;
4478
4479                 instr->regarray.regarray_id = r->id;
4480                 instr->regarray.idx_val = idx_val;
4481                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4482                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4483                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4484                 return 0;
4485         }
4486
4487         /* REGADD_RII. */
4488         src_val = strtoull(src, &src, 0);
4489         CHECK(!src[0], EINVAL);
4490
4491         idx_val = strtoul(idx, &idx, 0);
4492         CHECK(!idx[0], EINVAL);
4493
4494         instr->type = INSTR_REGADD_RII;
4495         instr->regarray.idx_val = idx_val;
4496         instr->regarray.dstsrc_val = src_val;
4497         return 0;
4498 }
4499
4500 static inline void
4501 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4502 {
4503         struct thread *t = &p->threads[p->thread_id];
4504         struct instruction *ip = t->ip;
4505
4506         /* Structs. */
4507         __instr_regprefetch_rh_exec(p, t, ip);
4508
4509         /* Thread. */
4510         thread_ip_inc(p);
4511 }
4512
4513 static inline void
4514 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4515 {
4516         struct thread *t = &p->threads[p->thread_id];
4517         struct instruction *ip = t->ip;
4518
4519         /* Structs. */
4520         __instr_regprefetch_rm_exec(p, t, ip);
4521
4522         /* Thread. */
4523         thread_ip_inc(p);
4524 }
4525
4526 static inline void
4527 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4528 {
4529         struct thread *t = &p->threads[p->thread_id];
4530         struct instruction *ip = t->ip;
4531
4532         /* Structs. */
4533         __instr_regprefetch_ri_exec(p, t, ip);
4534
4535         /* Thread. */
4536         thread_ip_inc(p);
4537 }
4538
4539 static inline void
4540 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4541 {
4542         struct thread *t = &p->threads[p->thread_id];
4543         struct instruction *ip = t->ip;
4544
4545         /* Structs. */
4546         __instr_regrd_hrh_exec(p, t, ip);
4547
4548         /* Thread. */
4549         thread_ip_inc(p);
4550 }
4551
4552 static inline void
4553 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4554 {
4555         struct thread *t = &p->threads[p->thread_id];
4556         struct instruction *ip = t->ip;
4557
4558         /* Structs. */
4559         __instr_regrd_hrm_exec(p, t, ip);
4560
4561         /* Thread. */
4562         thread_ip_inc(p);
4563 }
4564
4565 static inline void
4566 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4567 {
4568         struct thread *t = &p->threads[p->thread_id];
4569         struct instruction *ip = t->ip;
4570
4571         /* Structs. */
4572         __instr_regrd_mrh_exec(p, t, ip);
4573
4574         /* Thread. */
4575         thread_ip_inc(p);
4576 }
4577
4578 static inline void
4579 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4580 {
4581         struct thread *t = &p->threads[p->thread_id];
4582         struct instruction *ip = t->ip;
4583
4584         /* Structs. */
4585         __instr_regrd_mrm_exec(p, t, ip);
4586
4587         /* Thread. */
4588         thread_ip_inc(p);
4589 }
4590
4591 static inline void
4592 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4593 {
4594         struct thread *t = &p->threads[p->thread_id];
4595         struct instruction *ip = t->ip;
4596
4597         /* Structs. */
4598         __instr_regrd_hri_exec(p, t, ip);
4599
4600         /* Thread. */
4601         thread_ip_inc(p);
4602 }
4603
4604 static inline void
4605 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4606 {
4607         struct thread *t = &p->threads[p->thread_id];
4608         struct instruction *ip = t->ip;
4609
4610         /* Structs. */
4611         __instr_regrd_mri_exec(p, t, ip);
4612
4613         /* Thread. */
4614         thread_ip_inc(p);
4615 }
4616
4617 static inline void
4618 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4619 {
4620         struct thread *t = &p->threads[p->thread_id];
4621         struct instruction *ip = t->ip;
4622
4623         /* Structs. */
4624         __instr_regwr_rhh_exec(p, t, ip);
4625
4626         /* Thread. */
4627         thread_ip_inc(p);
4628 }
4629
4630 static inline void
4631 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4632 {
4633         struct thread *t = &p->threads[p->thread_id];
4634         struct instruction *ip = t->ip;
4635
4636         /* Structs. */
4637         __instr_regwr_rhm_exec(p, t, ip);
4638
4639         /* Thread. */
4640         thread_ip_inc(p);
4641 }
4642
4643 static inline void
4644 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4645 {
4646         struct thread *t = &p->threads[p->thread_id];
4647         struct instruction *ip = t->ip;
4648
4649         /* Structs. */
4650         __instr_regwr_rmh_exec(p, t, ip);
4651
4652         /* Thread. */
4653         thread_ip_inc(p);
4654 }
4655
4656 static inline void
4657 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4658 {
4659         struct thread *t = &p->threads[p->thread_id];
4660         struct instruction *ip = t->ip;
4661
4662         /* Structs. */
4663         __instr_regwr_rmm_exec(p, t, ip);
4664
4665         /* Thread. */
4666         thread_ip_inc(p);
4667 }
4668
4669 static inline void
4670 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4671 {
4672         struct thread *t = &p->threads[p->thread_id];
4673         struct instruction *ip = t->ip;
4674
4675         /* Structs. */
4676         __instr_regwr_rhi_exec(p, t, ip);
4677
4678         /* Thread. */
4679         thread_ip_inc(p);
4680 }
4681
4682 static inline void
4683 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4684 {
4685         struct thread *t = &p->threads[p->thread_id];
4686         struct instruction *ip = t->ip;
4687
4688         /* Structs. */
4689         __instr_regwr_rmi_exec(p, t, ip);
4690
4691         /* Thread. */
4692         thread_ip_inc(p);
4693 }
4694
4695 static inline void
4696 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4697 {
4698         struct thread *t = &p->threads[p->thread_id];
4699         struct instruction *ip = t->ip;
4700
4701         /* Structs. */
4702         __instr_regwr_rih_exec(p, t, ip);
4703
4704         /* Thread. */
4705         thread_ip_inc(p);
4706 }
4707
4708 static inline void
4709 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4710 {
4711         struct thread *t = &p->threads[p->thread_id];
4712         struct instruction *ip = t->ip;
4713
4714         /* Structs. */
4715         __instr_regwr_rim_exec(p, t, ip);
4716
4717         /* Thread. */
4718         thread_ip_inc(p);
4719 }
4720
4721 static inline void
4722 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4723 {
4724         struct thread *t = &p->threads[p->thread_id];
4725         struct instruction *ip = t->ip;
4726
4727         /* Structs. */
4728         __instr_regwr_rii_exec(p, t, ip);
4729
4730         /* Thread. */
4731         thread_ip_inc(p);
4732 }
4733
4734 static inline void
4735 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4736 {
4737         struct thread *t = &p->threads[p->thread_id];
4738         struct instruction *ip = t->ip;
4739
4740         /* Structs. */
4741         __instr_regadd_rhh_exec(p, t, ip);
4742
4743         /* Thread. */
4744         thread_ip_inc(p);
4745 }
4746
4747 static inline void
4748 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4749 {
4750         struct thread *t = &p->threads[p->thread_id];
4751         struct instruction *ip = t->ip;
4752
4753         /* Structs. */
4754         __instr_regadd_rhm_exec(p, t, ip);
4755
4756         /* Thread. */
4757         thread_ip_inc(p);
4758 }
4759
4760 static inline void
4761 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4762 {
4763         struct thread *t = &p->threads[p->thread_id];
4764         struct instruction *ip = t->ip;
4765
4766         /* Structs. */
4767         __instr_regadd_rmh_exec(p, t, ip);
4768
4769         /* Thread. */
4770         thread_ip_inc(p);
4771 }
4772
4773 static inline void
4774 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4775 {
4776         struct thread *t = &p->threads[p->thread_id];
4777         struct instruction *ip = t->ip;
4778
4779         /* Structs. */
4780         __instr_regadd_rmm_exec(p, t, ip);
4781
4782         /* Thread. */
4783         thread_ip_inc(p);
4784 }
4785
4786 static inline void
4787 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4788 {
4789         struct thread *t = &p->threads[p->thread_id];
4790         struct instruction *ip = t->ip;
4791
4792         /* Structs. */
4793         __instr_regadd_rhi_exec(p, t, ip);
4794
4795         /* Thread. */
4796         thread_ip_inc(p);
4797 }
4798
4799 static inline void
4800 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4801 {
4802         struct thread *t = &p->threads[p->thread_id];
4803         struct instruction *ip = t->ip;
4804
4805         /* Structs. */
4806         __instr_regadd_rmi_exec(p, t, ip);
4807
4808         /* Thread. */
4809         thread_ip_inc(p);
4810 }
4811
4812 static inline void
4813 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4814 {
4815         struct thread *t = &p->threads[p->thread_id];
4816         struct instruction *ip = t->ip;
4817
4818         /* Structs. */
4819         __instr_regadd_rih_exec(p, t, ip);
4820
4821         /* Thread. */
4822         thread_ip_inc(p);
4823 }
4824
4825 static inline void
4826 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4827 {
4828         struct thread *t = &p->threads[p->thread_id];
4829         struct instruction *ip = t->ip;
4830
4831         /* Structs. */
4832         __instr_regadd_rim_exec(p, t, ip);
4833
4834         /* Thread. */
4835         thread_ip_inc(p);
4836 }
4837
4838 static inline void
4839 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4840 {
4841         struct thread *t = &p->threads[p->thread_id];
4842         struct instruction *ip = t->ip;
4843
4844         /* Structs. */
4845         __instr_regadd_rii_exec(p, t, ip);
4846
4847         /* Thread. */
4848         thread_ip_inc(p);
4849 }
4850
4851 /*
4852  * metarray.
4853  */
4854 static struct metarray *
4855 metarray_find(struct rte_swx_pipeline *p, const char *name);
4856
4857 static int
4858 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4859                             struct action *action,
4860                             char **tokens,
4861                             int n_tokens,
4862                             struct instruction *instr,
4863                             struct instruction_data *data __rte_unused)
4864 {
4865         char *metarray = tokens[1], *idx = tokens[2];
4866         struct metarray *m;
4867         struct field *fidx;
4868         uint32_t idx_struct_id, idx_val;
4869
4870         CHECK(n_tokens == 3, EINVAL);
4871
4872         m = metarray_find(p, metarray);
4873         CHECK(m, EINVAL);
4874
4875         /* METPREFETCH_H, METPREFETCH_M. */
4876         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4877         if (fidx) {
4878                 CHECK(!fidx->var_size, EINVAL);
4879
4880                 instr->type = INSTR_METPREFETCH_M;
4881                 if (idx[0] == 'h')
4882                         instr->type = INSTR_METPREFETCH_H;
4883
4884                 instr->meter.metarray_id = m->id;
4885                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4886                 instr->meter.idx.n_bits = fidx->n_bits;
4887                 instr->meter.idx.offset = fidx->offset / 8;
4888                 return 0;
4889         }
4890
4891         /* METPREFETCH_I. */
4892         idx_val = strtoul(idx, &idx, 0);
4893         CHECK(!idx[0], EINVAL);
4894
4895         instr->type = INSTR_METPREFETCH_I;
4896         instr->meter.metarray_id = m->id;
4897         instr->meter.idx_val = idx_val;
4898         return 0;
4899 }
4900
4901 static int
4902 instr_meter_translate(struct rte_swx_pipeline *p,
4903                       struct action *action,
4904                       char **tokens,
4905                       int n_tokens,
4906                       struct instruction *instr,
4907                       struct instruction_data *data __rte_unused)
4908 {
4909         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4910         char *color_in = tokens[4], *color_out = tokens[5];
4911         struct metarray *m;
4912         struct field *fidx, *flength, *fcin, *fcout;
4913         uint32_t idx_struct_id, length_struct_id;
4914         uint32_t color_in_struct_id, color_out_struct_id;
4915
4916         CHECK(n_tokens == 6, EINVAL);
4917
4918         m = metarray_find(p, metarray);
4919         CHECK(m, EINVAL);
4920
4921         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4922
4923         flength = struct_field_parse(p, action, length, &length_struct_id);
4924         CHECK(flength, EINVAL);
4925         CHECK(!flength->var_size, EINVAL);
4926
4927         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4928
4929         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4930         CHECK(fcout, EINVAL);
4931         CHECK(!fcout->var_size, EINVAL);
4932
4933         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4934         if (fidx && fcin) {
4935                 CHECK(!fidx->var_size, EINVAL);
4936                 CHECK(!fcin->var_size, EINVAL);
4937
4938                 instr->type = INSTR_METER_MMM;
4939                 if (idx[0] == 'h' && length[0] == 'h')
4940                         instr->type = INSTR_METER_HHM;
4941                 if (idx[0] == 'h' && length[0] != 'h')
4942                         instr->type = INSTR_METER_HMM;
4943                 if (idx[0] != 'h' && length[0] == 'h')
4944                         instr->type = INSTR_METER_MHM;
4945
4946                 instr->meter.metarray_id = m->id;
4947
4948                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4949                 instr->meter.idx.n_bits = fidx->n_bits;
4950                 instr->meter.idx.offset = fidx->offset / 8;
4951
4952                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4953                 instr->meter.length.n_bits = flength->n_bits;
4954                 instr->meter.length.offset = flength->offset / 8;
4955
4956                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4957                 instr->meter.color_in.n_bits = fcin->n_bits;
4958                 instr->meter.color_in.offset = fcin->offset / 8;
4959
4960                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4961                 instr->meter.color_out.n_bits = fcout->n_bits;
4962                 instr->meter.color_out.offset = fcout->offset / 8;
4963         }
4964
4965         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4966         if (fidx && !fcin) {
4967                 uint32_t color_in_val;
4968
4969                 CHECK(!fidx->var_size, EINVAL);
4970
4971                 color_in_val = strtoul(color_in, &color_in, 0);
4972                 CHECK(!color_in[0], EINVAL);
4973
4974                 instr->type = INSTR_METER_MMI;
4975                 if (idx[0] == 'h' && length[0] == 'h')
4976                         instr->type = INSTR_METER_HHI;
4977                 if (idx[0] == 'h' && length[0] != 'h')
4978                         instr->type = INSTR_METER_HMI;
4979                 if (idx[0] != 'h' && length[0] == 'h')
4980                         instr->type = INSTR_METER_MHI;
4981
4982                 instr->meter.metarray_id = m->id;
4983
4984                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4985                 instr->meter.idx.n_bits = fidx->n_bits;
4986                 instr->meter.idx.offset = fidx->offset / 8;
4987
4988                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4989                 instr->meter.length.n_bits = flength->n_bits;
4990                 instr->meter.length.offset = flength->offset / 8;
4991
4992                 instr->meter.color_in_val = color_in_val;
4993
4994                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4995                 instr->meter.color_out.n_bits = fcout->n_bits;
4996                 instr->meter.color_out.offset = fcout->offset / 8;
4997         }
4998
4999         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5000         if (!fidx && fcin) {
5001                 uint32_t idx_val;
5002
5003                 idx_val = strtoul(idx, &idx, 0);
5004                 CHECK(!idx[0], EINVAL);
5005
5006                 CHECK(!fcin->var_size, EINVAL);
5007
5008                 instr->type = INSTR_METER_IMM;
5009                 if (length[0] == 'h')
5010                         instr->type = INSTR_METER_IHM;
5011
5012                 instr->meter.metarray_id = m->id;
5013
5014                 instr->meter.idx_val = idx_val;
5015
5016                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5017                 instr->meter.length.n_bits = flength->n_bits;
5018                 instr->meter.length.offset = flength->offset / 8;
5019
5020                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5021                 instr->meter.color_in.n_bits = fcin->n_bits;
5022                 instr->meter.color_in.offset = fcin->offset / 8;
5023
5024                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5025                 instr->meter.color_out.n_bits = fcout->n_bits;
5026                 instr->meter.color_out.offset = fcout->offset / 8;
5027         }
5028
5029         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5030         if (!fidx && !fcin) {
5031                 uint32_t idx_val, color_in_val;
5032
5033                 idx_val = strtoul(idx, &idx, 0);
5034                 CHECK(!idx[0], EINVAL);
5035
5036                 color_in_val = strtoul(color_in, &color_in, 0);
5037                 CHECK(!color_in[0], EINVAL);
5038
5039                 instr->type = INSTR_METER_IMI;
5040                 if (length[0] == 'h')
5041                         instr->type = INSTR_METER_IHI;
5042
5043                 instr->meter.metarray_id = m->id;
5044
5045                 instr->meter.idx_val = idx_val;
5046
5047                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5048                 instr->meter.length.n_bits = flength->n_bits;
5049                 instr->meter.length.offset = flength->offset / 8;
5050
5051                 instr->meter.color_in_val = color_in_val;
5052
5053                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5054                 instr->meter.color_out.n_bits = fcout->n_bits;
5055                 instr->meter.color_out.offset = fcout->offset / 8;
5056         }
5057
5058         return 0;
5059 }
5060
5061 static inline void
5062 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5063 {
5064         struct thread *t = &p->threads[p->thread_id];
5065         struct instruction *ip = t->ip;
5066
5067         /* Structs. */
5068         __instr_metprefetch_h_exec(p, t, ip);
5069
5070         /* Thread. */
5071         thread_ip_inc(p);
5072 }
5073
5074 static inline void
5075 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5076 {
5077         struct thread *t = &p->threads[p->thread_id];
5078         struct instruction *ip = t->ip;
5079
5080         /* Structs. */
5081         __instr_metprefetch_m_exec(p, t, ip);
5082
5083         /* Thread. */
5084         thread_ip_inc(p);
5085 }
5086
5087 static inline void
5088 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5089 {
5090         struct thread *t = &p->threads[p->thread_id];
5091         struct instruction *ip = t->ip;
5092
5093         /* Structs. */
5094         __instr_metprefetch_i_exec(p, t, ip);
5095
5096         /* Thread. */
5097         thread_ip_inc(p);
5098 }
5099
5100 static inline void
5101 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5102 {
5103         struct thread *t = &p->threads[p->thread_id];
5104         struct instruction *ip = t->ip;
5105
5106         /* Structs. */
5107         __instr_meter_hhm_exec(p, t, ip);
5108
5109         /* Thread. */
5110         thread_ip_inc(p);
5111 }
5112
5113 static inline void
5114 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5115 {
5116         struct thread *t = &p->threads[p->thread_id];
5117         struct instruction *ip = t->ip;
5118
5119         /* Structs. */
5120         __instr_meter_hhi_exec(p, t, ip);
5121
5122         /* Thread. */
5123         thread_ip_inc(p);
5124 }
5125
5126 static inline void
5127 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5128 {
5129         struct thread *t = &p->threads[p->thread_id];
5130         struct instruction *ip = t->ip;
5131
5132         /* Structs. */
5133         __instr_meter_hmm_exec(p, t, ip);
5134
5135         /* Thread. */
5136         thread_ip_inc(p);
5137 }
5138
5139 static inline void
5140 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5141 {
5142         struct thread *t = &p->threads[p->thread_id];
5143         struct instruction *ip = t->ip;
5144
5145         /* Structs. */
5146         __instr_meter_hmi_exec(p, t, ip);
5147
5148         /* Thread. */
5149         thread_ip_inc(p);
5150 }
5151
5152 static inline void
5153 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5154 {
5155         struct thread *t = &p->threads[p->thread_id];
5156         struct instruction *ip = t->ip;
5157
5158         /* Structs. */
5159         __instr_meter_mhm_exec(p, t, ip);
5160
5161         /* Thread. */
5162         thread_ip_inc(p);
5163 }
5164
5165 static inline void
5166 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5167 {
5168         struct thread *t = &p->threads[p->thread_id];
5169         struct instruction *ip = t->ip;
5170
5171         /* Structs. */
5172         __instr_meter_mhi_exec(p, t, ip);
5173
5174         /* Thread. */
5175         thread_ip_inc(p);
5176 }
5177
5178 static inline void
5179 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5180 {
5181         struct thread *t = &p->threads[p->thread_id];
5182         struct instruction *ip = t->ip;
5183
5184         /* Structs. */
5185         __instr_meter_mmm_exec(p, t, ip);
5186
5187         /* Thread. */
5188         thread_ip_inc(p);
5189 }
5190
5191 static inline void
5192 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5193 {
5194         struct thread *t = &p->threads[p->thread_id];
5195         struct instruction *ip = t->ip;
5196
5197         /* Structs. */
5198         __instr_meter_mmi_exec(p, t, ip);
5199
5200         /* Thread. */
5201         thread_ip_inc(p);
5202 }
5203
5204 static inline void
5205 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5206 {
5207         struct thread *t = &p->threads[p->thread_id];
5208         struct instruction *ip = t->ip;
5209
5210         /* Structs. */
5211         __instr_meter_ihm_exec(p, t, ip);
5212
5213         /* Thread. */
5214         thread_ip_inc(p);
5215 }
5216
5217 static inline void
5218 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5219 {
5220         struct thread *t = &p->threads[p->thread_id];
5221         struct instruction *ip = t->ip;
5222
5223         /* Structs. */
5224         __instr_meter_ihi_exec(p, t, ip);
5225
5226         /* Thread. */
5227         thread_ip_inc(p);
5228 }
5229
5230 static inline void
5231 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5232 {
5233         struct thread *t = &p->threads[p->thread_id];
5234         struct instruction *ip = t->ip;
5235
5236         /* Structs. */
5237         __instr_meter_imm_exec(p, t, ip);
5238
5239         /* Thread. */
5240         thread_ip_inc(p);
5241 }
5242
5243 static inline void
5244 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5245 {
5246         struct thread *t = &p->threads[p->thread_id];
5247         struct instruction *ip = t->ip;
5248
5249         /* Structs. */
5250         __instr_meter_imi_exec(p, t, ip);
5251
5252         /* Thread. */
5253         thread_ip_inc(p);
5254 }
5255
5256 /*
5257  * jmp.
5258  */
5259 static int
5260 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5261                     struct action *action __rte_unused,
5262                     char **tokens,
5263                     int n_tokens,
5264                     struct instruction *instr,
5265                     struct instruction_data *data)
5266 {
5267         CHECK(n_tokens == 2, EINVAL);
5268
5269         strcpy(data->jmp_label, tokens[1]);
5270
5271         instr->type = INSTR_JMP;
5272         instr->jmp.ip = NULL; /* Resolved later. */
5273         return 0;
5274 }
5275
5276 static int
5277 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5278                           struct action *action __rte_unused,
5279                           char **tokens,
5280                           int n_tokens,
5281                           struct instruction *instr,
5282                           struct instruction_data *data)
5283 {
5284         struct header *h;
5285
5286         CHECK(n_tokens == 3, EINVAL);
5287
5288         strcpy(data->jmp_label, tokens[1]);
5289
5290         h = header_parse(p, tokens[2]);
5291         CHECK(h, EINVAL);
5292
5293         instr->type = INSTR_JMP_VALID;
5294         instr->jmp.ip = NULL; /* Resolved later. */
5295         instr->jmp.header_id = h->id;
5296         return 0;
5297 }
5298
5299 static int
5300 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5301                             struct action *action __rte_unused,
5302                             char **tokens,
5303                             int n_tokens,
5304                             struct instruction *instr,
5305                             struct instruction_data *data)
5306 {
5307         struct header *h;
5308
5309         CHECK(n_tokens == 3, EINVAL);
5310
5311         strcpy(data->jmp_label, tokens[1]);
5312
5313         h = header_parse(p, tokens[2]);
5314         CHECK(h, EINVAL);
5315
5316         instr->type = INSTR_JMP_INVALID;
5317         instr->jmp.ip = NULL; /* Resolved later. */
5318         instr->jmp.header_id = h->id;
5319         return 0;
5320 }
5321
5322 static int
5323 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5324                         struct action *action,
5325                         char **tokens,
5326                         int n_tokens,
5327                         struct instruction *instr,
5328                         struct instruction_data *data)
5329 {
5330         CHECK(!action, EINVAL);
5331         CHECK(n_tokens == 2, EINVAL);
5332
5333         strcpy(data->jmp_label, tokens[1]);
5334
5335         instr->type = INSTR_JMP_HIT;
5336         instr->jmp.ip = NULL; /* Resolved later. */
5337         return 0;
5338 }
5339
5340 static int
5341 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5342                          struct action *action,
5343                          char **tokens,
5344                          int n_tokens,
5345                          struct instruction *instr,
5346                          struct instruction_data *data)
5347 {
5348         CHECK(!action, EINVAL);
5349         CHECK(n_tokens == 2, EINVAL);
5350
5351         strcpy(data->jmp_label, tokens[1]);
5352
5353         instr->type = INSTR_JMP_MISS;
5354         instr->jmp.ip = NULL; /* Resolved later. */
5355         return 0;
5356 }
5357
5358 static int
5359 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5360                                struct action *action,
5361                                char **tokens,
5362                                int n_tokens,
5363                                struct instruction *instr,
5364                                struct instruction_data *data)
5365 {
5366         struct action *a;
5367
5368         CHECK(!action, EINVAL);
5369         CHECK(n_tokens == 3, EINVAL);
5370
5371         strcpy(data->jmp_label, tokens[1]);
5372
5373         a = action_find(p, tokens[2]);
5374         CHECK(a, EINVAL);
5375
5376         instr->type = INSTR_JMP_ACTION_HIT;
5377         instr->jmp.ip = NULL; /* Resolved later. */
5378         instr->jmp.action_id = a->id;
5379         return 0;
5380 }
5381
5382 static int
5383 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5384                                 struct action *action,
5385                                 char **tokens,
5386                                 int n_tokens,
5387                                 struct instruction *instr,
5388                                 struct instruction_data *data)
5389 {
5390         struct action *a;
5391
5392         CHECK(!action, EINVAL);
5393         CHECK(n_tokens == 3, EINVAL);
5394
5395         strcpy(data->jmp_label, tokens[1]);
5396
5397         a = action_find(p, tokens[2]);
5398         CHECK(a, EINVAL);
5399
5400         instr->type = INSTR_JMP_ACTION_MISS;
5401         instr->jmp.ip = NULL; /* Resolved later. */
5402         instr->jmp.action_id = a->id;
5403         return 0;
5404 }
5405
5406 static int
5407 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5408                        struct action *action,
5409                        char **tokens,
5410                        int n_tokens,
5411                        struct instruction *instr,
5412                        struct instruction_data *data)
5413 {
5414         char *a = tokens[2], *b = tokens[3];
5415         struct field *fa, *fb;
5416         uint64_t b_val;
5417         uint32_t a_struct_id, b_struct_id;
5418
5419         CHECK(n_tokens == 4, EINVAL);
5420
5421         strcpy(data->jmp_label, tokens[1]);
5422
5423         fa = struct_field_parse(p, action, a, &a_struct_id);
5424         CHECK(fa, EINVAL);
5425         CHECK(!fa->var_size, EINVAL);
5426
5427         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5428         fb = struct_field_parse(p, action, b, &b_struct_id);
5429         if (fb) {
5430                 CHECK(!fb->var_size, EINVAL);
5431
5432                 instr->type = INSTR_JMP_EQ;
5433                 if (a[0] != 'h' && b[0] == 'h')
5434                         instr->type = INSTR_JMP_EQ_MH;
5435                 if (a[0] == 'h' && b[0] != 'h')
5436                         instr->type = INSTR_JMP_EQ_HM;
5437                 if (a[0] == 'h' && b[0] == 'h')
5438                         instr->type = INSTR_JMP_EQ_HH;
5439                 instr->jmp.ip = NULL; /* Resolved later. */
5440
5441                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5442                 instr->jmp.a.n_bits = fa->n_bits;
5443                 instr->jmp.a.offset = fa->offset / 8;
5444                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5445                 instr->jmp.b.n_bits = fb->n_bits;
5446                 instr->jmp.b.offset = fb->offset / 8;
5447                 return 0;
5448         }
5449
5450         /* JMP_EQ_I. */
5451         b_val = strtoull(b, &b, 0);
5452         CHECK(!b[0], EINVAL);
5453
5454         if (a[0] == 'h')
5455                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5456
5457         instr->type = INSTR_JMP_EQ_I;
5458         instr->jmp.ip = NULL; /* Resolved later. */
5459         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5460         instr->jmp.a.n_bits = fa->n_bits;
5461         instr->jmp.a.offset = fa->offset / 8;
5462         instr->jmp.b_val = b_val;
5463         return 0;
5464 }
5465
5466 static int
5467 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5468                         struct action *action,
5469                         char **tokens,
5470                         int n_tokens,
5471                         struct instruction *instr,
5472                         struct instruction_data *data)
5473 {
5474         char *a = tokens[2], *b = tokens[3];
5475         struct field *fa, *fb;
5476         uint64_t b_val;
5477         uint32_t a_struct_id, b_struct_id;
5478
5479         CHECK(n_tokens == 4, EINVAL);
5480
5481         strcpy(data->jmp_label, tokens[1]);
5482
5483         fa = struct_field_parse(p, action, a, &a_struct_id);
5484         CHECK(fa, EINVAL);
5485         CHECK(!fa->var_size, EINVAL);
5486
5487         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5488         fb = struct_field_parse(p, action, b, &b_struct_id);
5489         if (fb) {
5490                 CHECK(!fb->var_size, EINVAL);
5491
5492                 instr->type = INSTR_JMP_NEQ;
5493                 if (a[0] != 'h' && b[0] == 'h')
5494                         instr->type = INSTR_JMP_NEQ_MH;
5495                 if (a[0] == 'h' && b[0] != 'h')
5496                         instr->type = INSTR_JMP_NEQ_HM;
5497                 if (a[0] == 'h' && b[0] == 'h')
5498                         instr->type = INSTR_JMP_NEQ_HH;
5499                 instr->jmp.ip = NULL; /* Resolved later. */
5500
5501                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5502                 instr->jmp.a.n_bits = fa->n_bits;
5503                 instr->jmp.a.offset = fa->offset / 8;
5504                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5505                 instr->jmp.b.n_bits = fb->n_bits;
5506                 instr->jmp.b.offset = fb->offset / 8;
5507                 return 0;
5508         }
5509
5510         /* JMP_NEQ_I. */
5511         b_val = strtoull(b, &b, 0);
5512         CHECK(!b[0], EINVAL);
5513
5514         if (a[0] == 'h')
5515                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5516
5517         instr->type = INSTR_JMP_NEQ_I;
5518         instr->jmp.ip = NULL; /* Resolved later. */
5519         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5520         instr->jmp.a.n_bits = fa->n_bits;
5521         instr->jmp.a.offset = fa->offset / 8;
5522         instr->jmp.b_val = b_val;
5523         return 0;
5524 }
5525
5526 static int
5527 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5528                        struct action *action,
5529                        char **tokens,
5530                        int n_tokens,
5531                        struct instruction *instr,
5532                        struct instruction_data *data)
5533 {
5534         char *a = tokens[2], *b = tokens[3];
5535         struct field *fa, *fb;
5536         uint64_t b_val;
5537         uint32_t a_struct_id, b_struct_id;
5538
5539         CHECK(n_tokens == 4, EINVAL);
5540
5541         strcpy(data->jmp_label, tokens[1]);
5542
5543         fa = struct_field_parse(p, action, a, &a_struct_id);
5544         CHECK(fa, EINVAL);
5545         CHECK(!fa->var_size, EINVAL);
5546
5547         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5548         fb = struct_field_parse(p, action, b, &b_struct_id);
5549         if (fb) {
5550                 CHECK(!fb->var_size, EINVAL);
5551
5552                 instr->type = INSTR_JMP_LT;
5553                 if (a[0] == 'h' && b[0] != 'h')
5554                         instr->type = INSTR_JMP_LT_HM;
5555                 if (a[0] != 'h' && b[0] == 'h')
5556                         instr->type = INSTR_JMP_LT_MH;
5557                 if (a[0] == 'h' && b[0] == 'h')
5558                         instr->type = INSTR_JMP_LT_HH;
5559                 instr->jmp.ip = NULL; /* Resolved later. */
5560
5561                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5562                 instr->jmp.a.n_bits = fa->n_bits;
5563                 instr->jmp.a.offset = fa->offset / 8;
5564                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5565                 instr->jmp.b.n_bits = fb->n_bits;
5566                 instr->jmp.b.offset = fb->offset / 8;
5567                 return 0;
5568         }
5569
5570         /* JMP_LT_MI, JMP_LT_HI. */
5571         b_val = strtoull(b, &b, 0);
5572         CHECK(!b[0], EINVAL);
5573
5574         instr->type = INSTR_JMP_LT_MI;
5575         if (a[0] == 'h')
5576                 instr->type = INSTR_JMP_LT_HI;
5577         instr->jmp.ip = NULL; /* Resolved later. */
5578
5579         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5580         instr->jmp.a.n_bits = fa->n_bits;
5581         instr->jmp.a.offset = fa->offset / 8;
5582         instr->jmp.b_val = b_val;
5583         return 0;
5584 }
5585
5586 static int
5587 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5588                        struct action *action,
5589                        char **tokens,
5590                        int n_tokens,
5591                        struct instruction *instr,
5592                        struct instruction_data *data)
5593 {
5594         char *a = tokens[2], *b = tokens[3];
5595         struct field *fa, *fb;
5596         uint64_t b_val;
5597         uint32_t a_struct_id, b_struct_id;
5598
5599         CHECK(n_tokens == 4, EINVAL);
5600
5601         strcpy(data->jmp_label, tokens[1]);
5602
5603         fa = struct_field_parse(p, action, a, &a_struct_id);
5604         CHECK(fa, EINVAL);
5605         CHECK(!fa->var_size, EINVAL);
5606
5607         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5608         fb = struct_field_parse(p, action, b, &b_struct_id);
5609         if (fb) {
5610                 CHECK(!fb->var_size, EINVAL);
5611
5612                 instr->type = INSTR_JMP_GT;
5613                 if (a[0] == 'h' && b[0] != 'h')
5614                         instr->type = INSTR_JMP_GT_HM;
5615                 if (a[0] != 'h' && b[0] == 'h')
5616                         instr->type = INSTR_JMP_GT_MH;
5617                 if (a[0] == 'h' && b[0] == 'h')
5618                         instr->type = INSTR_JMP_GT_HH;
5619                 instr->jmp.ip = NULL; /* Resolved later. */
5620
5621                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5622                 instr->jmp.a.n_bits = fa->n_bits;
5623                 instr->jmp.a.offset = fa->offset / 8;
5624                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5625                 instr->jmp.b.n_bits = fb->n_bits;
5626                 instr->jmp.b.offset = fb->offset / 8;
5627                 return 0;
5628         }
5629
5630         /* JMP_GT_MI, JMP_GT_HI. */
5631         b_val = strtoull(b, &b, 0);
5632         CHECK(!b[0], EINVAL);
5633
5634         instr->type = INSTR_JMP_GT_MI;
5635         if (a[0] == 'h')
5636                 instr->type = INSTR_JMP_GT_HI;
5637         instr->jmp.ip = NULL; /* Resolved later. */
5638
5639         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5640         instr->jmp.a.n_bits = fa->n_bits;
5641         instr->jmp.a.offset = fa->offset / 8;
5642         instr->jmp.b_val = b_val;
5643         return 0;
5644 }
5645
5646 static inline void
5647 instr_jmp_exec(struct rte_swx_pipeline *p)
5648 {
5649         struct thread *t = &p->threads[p->thread_id];
5650         struct instruction *ip = t->ip;
5651
5652         TRACE("[Thread %2u] jmp\n", p->thread_id);
5653
5654         thread_ip_set(t, ip->jmp.ip);
5655 }
5656
5657 static inline void
5658 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5659 {
5660         struct thread *t = &p->threads[p->thread_id];
5661         struct instruction *ip = t->ip;
5662         uint32_t header_id = ip->jmp.header_id;
5663
5664         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5665
5666         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5667 }
5668
5669 static inline void
5670 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5671 {
5672         struct thread *t = &p->threads[p->thread_id];
5673         struct instruction *ip = t->ip;
5674         uint32_t header_id = ip->jmp.header_id;
5675
5676         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5677
5678         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5679 }
5680
5681 static inline void
5682 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5683 {
5684         struct thread *t = &p->threads[p->thread_id];
5685         struct instruction *ip = t->ip;
5686         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5687
5688         TRACE("[Thread %2u] jmph\n", p->thread_id);
5689
5690         t->ip = ip_next[t->hit];
5691 }
5692
5693 static inline void
5694 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5695 {
5696         struct thread *t = &p->threads[p->thread_id];
5697         struct instruction *ip = t->ip;
5698         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5699
5700         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5701
5702         t->ip = ip_next[t->hit];
5703 }
5704
5705 static inline void
5706 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5707 {
5708         struct thread *t = &p->threads[p->thread_id];
5709         struct instruction *ip = t->ip;
5710
5711         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5712
5713         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5714 }
5715
5716 static inline void
5717 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5718 {
5719         struct thread *t = &p->threads[p->thread_id];
5720         struct instruction *ip = t->ip;
5721
5722         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5723
5724         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5725 }
5726
5727 static inline void
5728 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5729 {
5730         struct thread *t = &p->threads[p->thread_id];
5731         struct instruction *ip = t->ip;
5732
5733         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5734
5735         JMP_CMP(t, ip, ==);
5736 }
5737
5738 static inline void
5739 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5740 {
5741         struct thread *t = &p->threads[p->thread_id];
5742         struct instruction *ip = t->ip;
5743
5744         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5745
5746         JMP_CMP_MH(t, ip, ==);
5747 }
5748
5749 static inline void
5750 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5751 {
5752         struct thread *t = &p->threads[p->thread_id];
5753         struct instruction *ip = t->ip;
5754
5755         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5756
5757         JMP_CMP_HM(t, ip, ==);
5758 }
5759
5760 static inline void
5761 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5762 {
5763         struct thread *t = &p->threads[p->thread_id];
5764         struct instruction *ip = t->ip;
5765
5766         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5767
5768         JMP_CMP_HH_FAST(t, ip, ==);
5769 }
5770
5771 static inline void
5772 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5773 {
5774         struct thread *t = &p->threads[p->thread_id];
5775         struct instruction *ip = t->ip;
5776
5777         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5778
5779         JMP_CMP_I(t, ip, ==);
5780 }
5781
5782 static inline void
5783 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5784 {
5785         struct thread *t = &p->threads[p->thread_id];
5786         struct instruction *ip = t->ip;
5787
5788         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5789
5790         JMP_CMP(t, ip, !=);
5791 }
5792
5793 static inline void
5794 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5795 {
5796         struct thread *t = &p->threads[p->thread_id];
5797         struct instruction *ip = t->ip;
5798
5799         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5800
5801         JMP_CMP_MH(t, ip, !=);
5802 }
5803
5804 static inline void
5805 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5806 {
5807         struct thread *t = &p->threads[p->thread_id];
5808         struct instruction *ip = t->ip;
5809
5810         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5811
5812         JMP_CMP_HM(t, ip, !=);
5813 }
5814
5815 static inline void
5816 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5817 {
5818         struct thread *t = &p->threads[p->thread_id];
5819         struct instruction *ip = t->ip;
5820
5821         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5822
5823         JMP_CMP_HH_FAST(t, ip, !=);
5824 }
5825
5826 static inline void
5827 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5828 {
5829         struct thread *t = &p->threads[p->thread_id];
5830         struct instruction *ip = t->ip;
5831
5832         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5833
5834         JMP_CMP_I(t, ip, !=);
5835 }
5836
5837 static inline void
5838 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5839 {
5840         struct thread *t = &p->threads[p->thread_id];
5841         struct instruction *ip = t->ip;
5842
5843         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5844
5845         JMP_CMP(t, ip, <);
5846 }
5847
5848 static inline void
5849 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5850 {
5851         struct thread *t = &p->threads[p->thread_id];
5852         struct instruction *ip = t->ip;
5853
5854         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5855
5856         JMP_CMP_MH(t, ip, <);
5857 }
5858
5859 static inline void
5860 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5861 {
5862         struct thread *t = &p->threads[p->thread_id];
5863         struct instruction *ip = t->ip;
5864
5865         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5866
5867         JMP_CMP_HM(t, ip, <);
5868 }
5869
5870 static inline void
5871 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5872 {
5873         struct thread *t = &p->threads[p->thread_id];
5874         struct instruction *ip = t->ip;
5875
5876         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5877
5878         JMP_CMP_HH(t, ip, <);
5879 }
5880
5881 static inline void
5882 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5883 {
5884         struct thread *t = &p->threads[p->thread_id];
5885         struct instruction *ip = t->ip;
5886
5887         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5888
5889         JMP_CMP_MI(t, ip, <);
5890 }
5891
5892 static inline void
5893 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5894 {
5895         struct thread *t = &p->threads[p->thread_id];
5896         struct instruction *ip = t->ip;
5897
5898         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5899
5900         JMP_CMP_HI(t, ip, <);
5901 }
5902
5903 static inline void
5904 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5905 {
5906         struct thread *t = &p->threads[p->thread_id];
5907         struct instruction *ip = t->ip;
5908
5909         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5910
5911         JMP_CMP(t, ip, >);
5912 }
5913
5914 static inline void
5915 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5916 {
5917         struct thread *t = &p->threads[p->thread_id];
5918         struct instruction *ip = t->ip;
5919
5920         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5921
5922         JMP_CMP_MH(t, ip, >);
5923 }
5924
5925 static inline void
5926 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5927 {
5928         struct thread *t = &p->threads[p->thread_id];
5929         struct instruction *ip = t->ip;
5930
5931         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5932
5933         JMP_CMP_HM(t, ip, >);
5934 }
5935
5936 static inline void
5937 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5938 {
5939         struct thread *t = &p->threads[p->thread_id];
5940         struct instruction *ip = t->ip;
5941
5942         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5943
5944         JMP_CMP_HH(t, ip, >);
5945 }
5946
5947 static inline void
5948 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5949 {
5950         struct thread *t = &p->threads[p->thread_id];
5951         struct instruction *ip = t->ip;
5952
5953         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5954
5955         JMP_CMP_MI(t, ip, >);
5956 }
5957
5958 static inline void
5959 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5960 {
5961         struct thread *t = &p->threads[p->thread_id];
5962         struct instruction *ip = t->ip;
5963
5964         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5965
5966         JMP_CMP_HI(t, ip, >);
5967 }
5968
5969 /*
5970  * return.
5971  */
5972 static int
5973 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5974                        struct action *action,
5975                        char **tokens __rte_unused,
5976                        int n_tokens,
5977                        struct instruction *instr,
5978                        struct instruction_data *data __rte_unused)
5979 {
5980         CHECK(action, EINVAL);
5981         CHECK(n_tokens == 1, EINVAL);
5982
5983         instr->type = INSTR_RETURN;
5984         return 0;
5985 }
5986
5987 static inline void
5988 instr_return_exec(struct rte_swx_pipeline *p)
5989 {
5990         struct thread *t = &p->threads[p->thread_id];
5991
5992         TRACE("[Thread %2u] return\n", p->thread_id);
5993
5994         t->ip = t->ret;
5995 }
5996
5997 static int
5998 instr_translate(struct rte_swx_pipeline *p,
5999                 struct action *action,
6000                 char *string,
6001                 struct instruction *instr,
6002                 struct instruction_data *data)
6003 {
6004         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6005         int n_tokens = 0, tpos = 0;
6006
6007         /* Parse the instruction string into tokens. */
6008         for ( ; ; ) {
6009                 char *token;
6010
6011                 token = strtok_r(string, " \t\v", &string);
6012                 if (!token)
6013                         break;
6014
6015                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6016                 CHECK_NAME(token, EINVAL);
6017
6018                 tokens[n_tokens] = token;
6019                 n_tokens++;
6020         }
6021
6022         CHECK(n_tokens, EINVAL);
6023
6024         /* Handle the optional instruction label. */
6025         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6026                 strcpy(data->label, tokens[0]);
6027
6028                 tpos += 2;
6029                 CHECK(n_tokens - tpos, EINVAL);
6030         }
6031
6032         /* Identify the instruction type. */
6033         if (!strcmp(tokens[tpos], "rx"))
6034                 return instr_rx_translate(p,
6035                                           action,
6036                                           &tokens[tpos],
6037                                           n_tokens - tpos,
6038                                           instr,
6039                                           data);
6040
6041         if (!strcmp(tokens[tpos], "tx"))
6042                 return instr_tx_translate(p,
6043                                           action,
6044                                           &tokens[tpos],
6045                                           n_tokens - tpos,
6046                                           instr,
6047                                           data);
6048
6049         if (!strcmp(tokens[tpos], "drop"))
6050                 return instr_drop_translate(p,
6051                                             action,
6052                                             &tokens[tpos],
6053                                             n_tokens - tpos,
6054                                             instr,
6055                                             data);
6056
6057         if (!strcmp(tokens[tpos], "mirror"))
6058                 return instr_mirror_translate(p,
6059                                               action,
6060                                               &tokens[tpos],
6061                                               n_tokens - tpos,
6062                                               instr,
6063                                               data);
6064
6065         if (!strcmp(tokens[tpos], "recirculate"))
6066                 return instr_recirculate_translate(p,
6067                                               action,
6068                                               &tokens[tpos],
6069                                               n_tokens - tpos,
6070                                               instr,
6071                                               data);
6072
6073         if (!strcmp(tokens[tpos], "recircid"))
6074                 return instr_recircid_translate(p,
6075                                               action,
6076                                               &tokens[tpos],
6077                                               n_tokens - tpos,
6078                                               instr,
6079                                               data);
6080
6081         if (!strcmp(tokens[tpos], "extract"))
6082                 return instr_hdr_extract_translate(p,
6083                                                    action,
6084                                                    &tokens[tpos],
6085                                                    n_tokens - tpos,
6086                                                    instr,
6087                                                    data);
6088
6089         if (!strcmp(tokens[tpos], "lookahead"))
6090                 return instr_hdr_lookahead_translate(p,
6091                                                      action,
6092                                                      &tokens[tpos],
6093                                                      n_tokens - tpos,
6094                                                      instr,
6095                                                      data);
6096
6097         if (!strcmp(tokens[tpos], "emit"))
6098                 return instr_hdr_emit_translate(p,
6099                                                 action,
6100                                                 &tokens[tpos],
6101                                                 n_tokens - tpos,
6102                                                 instr,
6103                                                 data);
6104
6105         if (!strcmp(tokens[tpos], "validate"))
6106                 return instr_hdr_validate_translate(p,
6107                                                     action,
6108                                                     &tokens[tpos],
6109                                                     n_tokens - tpos,
6110                                                     instr,
6111                                                     data);
6112
6113         if (!strcmp(tokens[tpos], "invalidate"))
6114                 return instr_hdr_invalidate_translate(p,
6115                                                       action,
6116                                                       &tokens[tpos],
6117                                                       n_tokens - tpos,
6118                                                       instr,
6119                                                       data);
6120
6121         if (!strcmp(tokens[tpos], "mov"))
6122                 return instr_mov_translate(p,
6123                                            action,
6124                                            &tokens[tpos],
6125                                            n_tokens - tpos,
6126                                            instr,
6127                                            data);
6128
6129         if (!strcmp(tokens[tpos], "add"))
6130                 return instr_alu_add_translate(p,
6131                                                action,
6132                                                &tokens[tpos],
6133                                                n_tokens - tpos,
6134                                                instr,
6135                                                data);
6136
6137         if (!strcmp(tokens[tpos], "sub"))
6138                 return instr_alu_sub_translate(p,
6139                                                action,
6140                                                &tokens[tpos],
6141                                                n_tokens - tpos,
6142                                                instr,
6143                                                data);
6144
6145         if (!strcmp(tokens[tpos], "ckadd"))
6146                 return instr_alu_ckadd_translate(p,
6147                                                  action,
6148                                                  &tokens[tpos],
6149                                                  n_tokens - tpos,
6150                                                  instr,
6151                                                  data);
6152
6153         if (!strcmp(tokens[tpos], "cksub"))
6154                 return instr_alu_cksub_translate(p,
6155                                                  action,
6156                                                  &tokens[tpos],
6157                                                  n_tokens - tpos,
6158                                                  instr,
6159                                                  data);
6160
6161         if (!strcmp(tokens[tpos], "and"))
6162                 return instr_alu_and_translate(p,
6163                                                action,
6164                                                &tokens[tpos],
6165                                                n_tokens - tpos,
6166                                                instr,
6167                                                data);
6168
6169         if (!strcmp(tokens[tpos], "or"))
6170                 return instr_alu_or_translate(p,
6171                                               action,
6172                                               &tokens[tpos],
6173                                               n_tokens - tpos,
6174                                               instr,
6175                                               data);
6176
6177         if (!strcmp(tokens[tpos], "xor"))
6178                 return instr_alu_xor_translate(p,
6179                                                action,
6180                                                &tokens[tpos],
6181                                                n_tokens - tpos,
6182                                                instr,
6183                                                data);
6184
6185         if (!strcmp(tokens[tpos], "shl"))
6186                 return instr_alu_shl_translate(p,
6187                                                action,
6188                                                &tokens[tpos],
6189                                                n_tokens - tpos,
6190                                                instr,
6191                                                data);
6192
6193         if (!strcmp(tokens[tpos], "shr"))
6194                 return instr_alu_shr_translate(p,
6195                                                action,
6196                                                &tokens[tpos],
6197                                                n_tokens - tpos,
6198                                                instr,
6199                                                data);
6200
6201         if (!strcmp(tokens[tpos], "regprefetch"))
6202                 return instr_regprefetch_translate(p,
6203                                                    action,
6204                                                    &tokens[tpos],
6205                                                    n_tokens - tpos,
6206                                                    instr,
6207                                                    data);
6208
6209         if (!strcmp(tokens[tpos], "regrd"))
6210                 return instr_regrd_translate(p,
6211                                              action,
6212                                              &tokens[tpos],
6213                                              n_tokens - tpos,
6214                                              instr,
6215                                              data);
6216
6217         if (!strcmp(tokens[tpos], "regwr"))
6218                 return instr_regwr_translate(p,
6219                                              action,
6220                                              &tokens[tpos],
6221                                              n_tokens - tpos,
6222                                              instr,
6223                                              data);
6224
6225         if (!strcmp(tokens[tpos], "regadd"))
6226                 return instr_regadd_translate(p,
6227                                               action,
6228                                               &tokens[tpos],
6229                                               n_tokens - tpos,
6230                                               instr,
6231                                               data);
6232
6233         if (!strcmp(tokens[tpos], "metprefetch"))
6234                 return instr_metprefetch_translate(p,
6235                                                    action,
6236                                                    &tokens[tpos],
6237                                                    n_tokens - tpos,
6238                                                    instr,
6239                                                    data);
6240
6241         if (!strcmp(tokens[tpos], "meter"))
6242                 return instr_meter_translate(p,
6243                                              action,
6244                                              &tokens[tpos],
6245                                              n_tokens - tpos,
6246                                              instr,
6247                                              data);
6248
6249         if (!strcmp(tokens[tpos], "table"))
6250                 return instr_table_translate(p,
6251                                              action,
6252                                              &tokens[tpos],
6253                                              n_tokens - tpos,
6254                                              instr,
6255                                              data);
6256
6257         if (!strcmp(tokens[tpos], "learn"))
6258                 return instr_learn_translate(p,
6259                                              action,
6260                                              &tokens[tpos],
6261                                              n_tokens - tpos,
6262                                              instr,
6263                                              data);
6264         if (!strcmp(tokens[tpos], "rearm"))
6265                 return instr_rearm_translate(p,
6266                                              action,
6267                                              &tokens[tpos],
6268                                              n_tokens - tpos,
6269                                              instr,
6270                                              data);
6271
6272         if (!strcmp(tokens[tpos], "forget"))
6273                 return instr_forget_translate(p,
6274                                               action,
6275                                               &tokens[tpos],
6276                                               n_tokens - tpos,
6277                                               instr,
6278                                               data);
6279
6280         if (!strcmp(tokens[tpos], "extern"))
6281                 return instr_extern_translate(p,
6282                                               action,
6283                                               &tokens[tpos],
6284                                               n_tokens - tpos,
6285                                               instr,
6286                                               data);
6287
6288         if (!strcmp(tokens[tpos], "hash"))
6289                 return instr_hash_translate(p,
6290                                             action,
6291                                             &tokens[tpos],
6292                                             n_tokens - tpos,
6293                                             instr,
6294                                             data);
6295
6296         if (!strcmp(tokens[tpos], "jmp"))
6297                 return instr_jmp_translate(p,
6298                                            action,
6299                                            &tokens[tpos],
6300                                            n_tokens - tpos,
6301                                            instr,
6302                                            data);
6303
6304         if (!strcmp(tokens[tpos], "jmpv"))
6305                 return instr_jmp_valid_translate(p,
6306                                                  action,
6307                                                  &tokens[tpos],
6308                                                  n_tokens - tpos,
6309                                                  instr,
6310                                                  data);
6311
6312         if (!strcmp(tokens[tpos], "jmpnv"))
6313                 return instr_jmp_invalid_translate(p,
6314                                                    action,
6315                                                    &tokens[tpos],
6316                                                    n_tokens - tpos,
6317                                                    instr,
6318                                                    data);
6319
6320         if (!strcmp(tokens[tpos], "jmph"))
6321                 return instr_jmp_hit_translate(p,
6322                                                action,
6323                                                &tokens[tpos],
6324                                                n_tokens - tpos,
6325                                                instr,
6326                                                data);
6327
6328         if (!strcmp(tokens[tpos], "jmpnh"))
6329                 return instr_jmp_miss_translate(p,
6330                                                 action,
6331                                                 &tokens[tpos],
6332                                                 n_tokens - tpos,
6333                                                 instr,
6334                                                 data);
6335
6336         if (!strcmp(tokens[tpos], "jmpa"))
6337                 return instr_jmp_action_hit_translate(p,
6338                                                       action,
6339                                                       &tokens[tpos],
6340                                                       n_tokens - tpos,
6341                                                       instr,
6342                                                       data);
6343
6344         if (!strcmp(tokens[tpos], "jmpna"))
6345                 return instr_jmp_action_miss_translate(p,
6346                                                        action,
6347                                                        &tokens[tpos],
6348                                                        n_tokens - tpos,
6349                                                        instr,
6350                                                        data);
6351
6352         if (!strcmp(tokens[tpos], "jmpeq"))
6353                 return instr_jmp_eq_translate(p,
6354                                               action,
6355                                               &tokens[tpos],
6356                                               n_tokens - tpos,
6357                                               instr,
6358                                               data);
6359
6360         if (!strcmp(tokens[tpos], "jmpneq"))
6361                 return instr_jmp_neq_translate(p,
6362                                                action,
6363                                                &tokens[tpos],
6364                                                n_tokens - tpos,
6365                                                instr,
6366                                                data);
6367
6368         if (!strcmp(tokens[tpos], "jmplt"))
6369                 return instr_jmp_lt_translate(p,
6370                                               action,
6371                                               &tokens[tpos],
6372                                               n_tokens - tpos,
6373                                               instr,
6374                                               data);
6375
6376         if (!strcmp(tokens[tpos], "jmpgt"))
6377                 return instr_jmp_gt_translate(p,
6378                                               action,
6379                                               &tokens[tpos],
6380                                               n_tokens - tpos,
6381                                               instr,
6382                                               data);
6383
6384         if (!strcmp(tokens[tpos], "return"))
6385                 return instr_return_translate(p,
6386                                               action,
6387                                               &tokens[tpos],
6388                                               n_tokens - tpos,
6389                                               instr,
6390                                               data);
6391
6392         return -EINVAL;
6393 }
6394
6395 static struct instruction_data *
6396 label_find(struct instruction_data *data, uint32_t n, const char *label)
6397 {
6398         uint32_t i;
6399
6400         for (i = 0; i < n; i++)
6401                 if (!strcmp(label, data[i].label))
6402                         return &data[i];
6403
6404         return NULL;
6405 }
6406
6407 static uint32_t
6408 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6409 {
6410         uint32_t count = 0, i;
6411
6412         if (!label[0])
6413                 return 0;
6414
6415         for (i = 0; i < n; i++)
6416                 if (!strcmp(label, data[i].jmp_label))
6417                         count++;
6418
6419         return count;
6420 }
6421
6422 static int
6423 instr_label_check(struct instruction_data *instruction_data,
6424                   uint32_t n_instructions)
6425 {
6426         uint32_t i;
6427
6428         /* Check that all instruction labels are unique. */
6429         for (i = 0; i < n_instructions; i++) {
6430                 struct instruction_data *data = &instruction_data[i];
6431                 char *label = data->label;
6432                 uint32_t j;
6433
6434                 if (!label[0])
6435                         continue;
6436
6437                 for (j = i + 1; j < n_instructions; j++)
6438                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6439         }
6440
6441         /* Check that no jump instruction (either conditional or not) can jump to itself (loop). */
6442         for (i = 0; i < n_instructions; i++) {
6443                 struct instruction_data *data = &instruction_data[i];
6444                 char *label = data->label;
6445                 char *jmp_label = data->jmp_label;
6446
6447                 /* Continue if this instruction does not have a label or it is not a jump. */
6448                 if (!label[0] || !jmp_label[0])
6449                         continue;
6450
6451                 CHECK(strcmp(label, jmp_label), EINVAL);
6452         }
6453
6454         /* Get users for each instruction label. */
6455         for (i = 0; i < n_instructions; i++) {
6456                 struct instruction_data *data = &instruction_data[i];
6457                 char *label = data->label;
6458
6459                 data->n_users = label_is_used(instruction_data,
6460                                               n_instructions,
6461                                               label);
6462         }
6463
6464         return 0;
6465 }
6466
6467 static int
6468 instr_jmp_resolve(struct instruction *instructions,
6469                   struct instruction_data *instruction_data,
6470                   uint32_t n_instructions)
6471 {
6472         uint32_t i;
6473
6474         for (i = 0; i < n_instructions; i++) {
6475                 struct instruction *instr = &instructions[i];
6476                 struct instruction_data *data = &instruction_data[i];
6477                 struct instruction_data *found;
6478
6479                 if (!instruction_is_jmp(instr))
6480                         continue;
6481
6482                 found = label_find(instruction_data,
6483                                    n_instructions,
6484                                    data->jmp_label);
6485                 CHECK(found, EINVAL);
6486
6487                 instr->jmp.ip = &instructions[found - instruction_data];
6488         }
6489
6490         return 0;
6491 }
6492
6493 static int
6494 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6495              struct action *a,
6496              struct instruction *instr,
6497              struct instruction_data *data __rte_unused,
6498              uint32_t n_instructions)
6499 {
6500         if (!a) {
6501                 enum instruction_type type;
6502                 uint32_t i;
6503
6504                 /* Check that the first instruction is rx. */
6505                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6506
6507                 /* Check that there is at least one tx instruction. */
6508                 for (i = 0; i < n_instructions; i++) {
6509                         type = instr[i].type;
6510
6511                         if (instruction_is_tx(type))
6512                                 break;
6513                 }
6514                 CHECK(i < n_instructions, EINVAL);
6515
6516                 /* Check that the last instruction is either tx or unconditional
6517                  * jump.
6518                  */
6519                 type = instr[n_instructions - 1].type;
6520                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6521         }
6522
6523         if (a) {
6524                 enum instruction_type type;
6525                 uint32_t i;
6526
6527                 /* Check that there is at least one return or tx instruction. */
6528                 for (i = 0; i < n_instructions; i++) {
6529                         type = instr[i].type;
6530
6531                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6532                                 break;
6533                 }
6534                 CHECK(i < n_instructions, EINVAL);
6535         }
6536
6537         return 0;
6538 }
6539
6540 static uint32_t
6541 instr_compact(struct instruction *instructions,
6542               struct instruction_data *instruction_data,
6543               uint32_t n_instructions)
6544 {
6545         uint32_t i, pos = 0;
6546
6547         /* Eliminate the invalid instructions that have been optimized out. */
6548         for (i = 0; i < n_instructions; i++) {
6549                 struct instruction *instr = &instructions[i];
6550                 struct instruction_data *data = &instruction_data[i];
6551
6552                 if (data->invalid)
6553                         continue;
6554
6555                 if (i != pos) {
6556                         memcpy(&instructions[pos], instr, sizeof(*instr));
6557                         memcpy(&instruction_data[pos], data, sizeof(*data));
6558                 }
6559
6560                 pos++;
6561         }
6562
6563         return pos;
6564 }
6565
6566 static int
6567 instr_pattern_extract_many_search(struct instruction *instr,
6568                                   struct instruction_data *data,
6569                                   uint32_t n_instr,
6570                                   uint32_t *n_pattern_instr)
6571 {
6572         uint32_t i;
6573
6574         for (i = 0; i < n_instr; i++) {
6575                 if (data[i].invalid)
6576                         break;
6577
6578                 if (instr[i].type != INSTR_HDR_EXTRACT)
6579                         break;
6580
6581                 if (i == RTE_DIM(instr->io.hdr.header_id))
6582                         break;
6583
6584                 if (i && data[i].n_users)
6585                         break;
6586         }
6587
6588         if (i < 2)
6589                 return 0;
6590
6591         *n_pattern_instr = i;
6592         return 1;
6593 }
6594
6595 static void
6596 instr_pattern_extract_many_replace(struct instruction *instr,
6597                                    struct instruction_data *data,
6598                                    uint32_t n_instr)
6599 {
6600         uint32_t i;
6601
6602         for (i = 1; i < n_instr; i++) {
6603                 instr[0].type++;
6604                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6605                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6606                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6607
6608                 data[i].invalid = 1;
6609         }
6610 }
6611
6612 static uint32_t
6613 instr_pattern_extract_many_optimize(struct instruction *instructions,
6614                                     struct instruction_data *instruction_data,
6615                                     uint32_t n_instructions)
6616 {
6617         uint32_t i;
6618
6619         for (i = 0; i < n_instructions; ) {
6620                 struct instruction *instr = &instructions[i];
6621                 struct instruction_data *data = &instruction_data[i];
6622                 uint32_t n_instr = 0;
6623                 int detected;
6624
6625                 /* Extract many. */
6626                 detected = instr_pattern_extract_many_search(instr,
6627                                                              data,
6628                                                              n_instructions - i,
6629                                                              &n_instr);
6630                 if (detected) {
6631                         instr_pattern_extract_many_replace(instr,
6632                                                            data,
6633                                                            n_instr);
6634                         i += n_instr;
6635                         continue;
6636                 }
6637
6638                 /* No pattern starting at the current instruction. */
6639                 i++;
6640         }
6641
6642         /* Eliminate the invalid instructions that have been optimized out. */
6643         n_instructions = instr_compact(instructions,
6644                                        instruction_data,
6645                                        n_instructions);
6646
6647         return n_instructions;
6648 }
6649
6650 static int
6651 instr_pattern_emit_many_tx_search(struct instruction *instr,
6652                                   struct instruction_data *data,
6653                                   uint32_t n_instr,
6654                                   uint32_t *n_pattern_instr)
6655 {
6656         uint32_t i;
6657
6658         for (i = 0; i < n_instr; i++) {
6659                 if (data[i].invalid)
6660                         break;
6661
6662                 if (instr[i].type != INSTR_HDR_EMIT)
6663                         break;
6664
6665                 if (i == RTE_DIM(instr->io.hdr.header_id))
6666                         break;
6667
6668                 if (i && data[i].n_users)
6669                         break;
6670         }
6671
6672         if (!i)
6673                 return 0;
6674
6675         if (instr[i].type != INSTR_TX)
6676                 return 0;
6677
6678         if (data[i].n_users)
6679                 return 0;
6680
6681         i++;
6682
6683         *n_pattern_instr = i;
6684         return 1;
6685 }
6686
6687 static void
6688 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6689                                    struct instruction_data *data,
6690                                    uint32_t n_instr)
6691 {
6692         uint32_t i;
6693
6694         /* Any emit instruction in addition to the first one. */
6695         for (i = 1; i < n_instr - 1; i++) {
6696                 instr[0].type++;
6697                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6698                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6699                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6700
6701                 data[i].invalid = 1;
6702         }
6703
6704         /* The TX instruction is the last one in the pattern. */
6705         instr[0].type++;
6706         instr[0].io.io.offset = instr[i].io.io.offset;
6707         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6708         data[i].invalid = 1;
6709 }
6710
6711 static uint32_t
6712 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6713                                     struct instruction_data *instruction_data,
6714                                     uint32_t n_instructions)
6715 {
6716         uint32_t i;
6717
6718         for (i = 0; i < n_instructions; ) {
6719                 struct instruction *instr = &instructions[i];
6720                 struct instruction_data *data = &instruction_data[i];
6721                 uint32_t n_instr = 0;
6722                 int detected;
6723
6724                 /* Emit many + TX. */
6725                 detected = instr_pattern_emit_many_tx_search(instr,
6726                                                              data,
6727                                                              n_instructions - i,
6728                                                              &n_instr);
6729                 if (detected) {
6730                         instr_pattern_emit_many_tx_replace(instr,
6731                                                            data,
6732                                                            n_instr);
6733                         i += n_instr;
6734                         continue;
6735                 }
6736
6737                 /* No pattern starting at the current instruction. */
6738                 i++;
6739         }
6740
6741         /* Eliminate the invalid instructions that have been optimized out. */
6742         n_instructions = instr_compact(instructions,
6743                                        instruction_data,
6744                                        n_instructions);
6745
6746         return n_instructions;
6747 }
6748
6749 static uint32_t
6750 action_arg_src_mov_count(struct action *a,
6751                          uint32_t arg_id,
6752                          struct instruction *instructions,
6753                          struct instruction_data *instruction_data,
6754                          uint32_t n_instructions);
6755
6756 static int
6757 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6758                                       struct action *a,
6759                                       struct instruction *instr,
6760                                       struct instruction_data *data,
6761                                       uint32_t n_instr,
6762                                       struct instruction *instructions,
6763                                       struct instruction_data *instruction_data,
6764                                       uint32_t n_instructions,
6765                                       uint32_t *n_pattern_instr)
6766 {
6767         struct header *h;
6768         uint32_t src_field_id, i, j;
6769
6770         /* Prerequisites. */
6771         if (!a || !a->st)
6772                 return 0;
6773
6774         /* First instruction: MOV_HM. */
6775         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6776                 return 0;
6777
6778         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6779         if (!h || h->st->var_size)
6780                 return 0;
6781
6782         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6783                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6784                         break;
6785
6786         if (src_field_id == a->st->n_fields)
6787                 return 0;
6788
6789         if (instr[0].mov.dst.offset ||
6790             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6791             instr[0].mov.src.struct_id ||
6792             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6793             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6794                 return 0;
6795
6796         if ((n_instr < h->st->n_fields + 1) ||
6797              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6798                 return 0;
6799
6800         /* Subsequent instructions: MOV_HM. */
6801         for (i = 1; i < h->st->n_fields; i++)
6802                 if (data[i].invalid ||
6803                     data[i].n_users ||
6804                     (instr[i].type != INSTR_MOV_HM) ||
6805                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6806                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6807                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6808                     instr[i].mov.src.struct_id ||
6809                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6810                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6811                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6812                         return 0;
6813
6814         /* Last instruction: HDR_VALIDATE. */
6815         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6816             (instr[i].valid.header_id != h->id))
6817                 return 0;
6818
6819         /* Check that none of the action args that are used as source for this
6820          * DMA transfer are not used as source in any other mov instruction.
6821          */
6822         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6823                 uint32_t n_users;
6824
6825                 n_users = action_arg_src_mov_count(a,
6826                                                    j,
6827                                                    instructions,
6828                                                    instruction_data,
6829                                                    n_instructions);
6830                 if (n_users > 1)
6831                         return 0;
6832         }
6833
6834         *n_pattern_instr = 1 + i;
6835         return 1;
6836 }
6837
6838 static void
6839 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6840                                        struct action *a,
6841                                        struct instruction *instr,
6842                                        struct instruction_data *data,
6843                                        uint32_t n_instr)
6844 {
6845         struct header *h;
6846         uint32_t src_field_id, src_offset, i;
6847
6848         /* Read from the instructions before they are modified. */
6849         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6850         if (!h)
6851                 return;
6852
6853         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6854                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6855                         break;
6856
6857         if (src_field_id == a->st->n_fields)
6858                 return;
6859
6860         src_offset = instr[0].mov.src.offset;
6861
6862         /* Modify the instructions. */
6863         instr[0].type = INSTR_DMA_HT;
6864         instr[0].dma.dst.header_id[0] = h->id;
6865         instr[0].dma.dst.struct_id[0] = h->struct_id;
6866         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6867         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6868
6869         for (i = 1; i < n_instr; i++)
6870                 data[i].invalid = 1;
6871
6872         /* Update the endianness of the action arguments to header endianness. */
6873         for (i = 0; i < h->st->n_fields; i++)
6874                 a->args_endianness[src_field_id + i] = 1;
6875 }
6876
6877 static uint32_t
6878 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6879                                         struct action *a,
6880                                         struct instruction *instructions,
6881                                         struct instruction_data *instruction_data,
6882                                         uint32_t n_instructions)
6883 {
6884         uint32_t i;
6885
6886         if (!a || !a->st)
6887                 return n_instructions;
6888
6889         for (i = 0; i < n_instructions; ) {
6890                 struct instruction *instr = &instructions[i];
6891                 struct instruction_data *data = &instruction_data[i];
6892                 uint32_t n_instr = 0;
6893                 int detected;
6894
6895                 /* Mov all + validate. */
6896                 detected = instr_pattern_mov_all_validate_search(p,
6897                                                                  a,
6898                                                                  instr,
6899                                                                  data,
6900                                                                  n_instructions - i,
6901                                                                  instructions,
6902                                                                  instruction_data,
6903                                                                  n_instructions,
6904                                                                  &n_instr);
6905                 if (detected) {
6906                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6907                         i += n_instr;
6908                         continue;
6909                 }
6910
6911                 /* No pattern starting at the current instruction. */
6912                 i++;
6913         }
6914
6915         /* Eliminate the invalid instructions that have been optimized out. */
6916         n_instructions = instr_compact(instructions,
6917                                        instruction_data,
6918                                        n_instructions);
6919
6920         return n_instructions;
6921 }
6922
6923 static int
6924 instr_pattern_dma_many_search(struct instruction *instr,
6925                               struct instruction_data *data,
6926                               uint32_t n_instr,
6927                               uint32_t *n_pattern_instr)
6928 {
6929         uint32_t i;
6930
6931         for (i = 0; i < n_instr; i++) {
6932                 if (data[i].invalid)
6933                         break;
6934
6935                 if (instr[i].type != INSTR_DMA_HT)
6936                         break;
6937
6938                 if (i == RTE_DIM(instr->dma.dst.header_id))
6939                         break;
6940
6941                 if (i && data[i].n_users)
6942                         break;
6943         }
6944
6945         if (i < 2)
6946                 return 0;
6947
6948         *n_pattern_instr = i;
6949         return 1;
6950 }
6951
6952 static void
6953 instr_pattern_dma_many_replace(struct instruction *instr,
6954                                struct instruction_data *data,
6955                                uint32_t n_instr)
6956 {
6957         uint32_t i;
6958
6959         for (i = 1; i < n_instr; i++) {
6960                 instr[0].type++;
6961                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6962                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6963                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6964                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6965
6966                 data[i].invalid = 1;
6967         }
6968 }
6969
6970 static uint32_t
6971 instr_pattern_dma_many_optimize(struct instruction *instructions,
6972                struct instruction_data *instruction_data,
6973                uint32_t n_instructions)
6974 {
6975         uint32_t i;
6976
6977         for (i = 0; i < n_instructions; ) {
6978                 struct instruction *instr = &instructions[i];
6979                 struct instruction_data *data = &instruction_data[i];
6980                 uint32_t n_instr = 0;
6981                 int detected;
6982
6983                 /* DMA many. */
6984                 detected = instr_pattern_dma_many_search(instr,
6985                                                          data,
6986                                                          n_instructions - i,
6987                                                          &n_instr);
6988                 if (detected) {
6989                         instr_pattern_dma_many_replace(instr, data, n_instr);
6990                         i += n_instr;
6991                         continue;
6992                 }
6993
6994                 /* No pattern starting at the current instruction. */
6995                 i++;
6996         }
6997
6998         /* Eliminate the invalid instructions that have been optimized out. */
6999         n_instructions = instr_compact(instructions,
7000                                        instruction_data,
7001                                        n_instructions);
7002
7003         return n_instructions;
7004 }
7005
7006 static uint32_t
7007 instr_optimize(struct rte_swx_pipeline *p,
7008                struct action *a,
7009                struct instruction *instructions,
7010                struct instruction_data *instruction_data,
7011                uint32_t n_instructions)
7012 {
7013         /* Extract many. */
7014         n_instructions = instr_pattern_extract_many_optimize(instructions,
7015                                                              instruction_data,
7016                                                              n_instructions);
7017
7018         /* Emit many + TX. */
7019         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7020                                                              instruction_data,
7021                                                              n_instructions);
7022
7023         /* Mov all + validate. */
7024         n_instructions = instr_pattern_mov_all_validate_optimize(p,
7025                                                                  a,
7026                                                                  instructions,
7027                                                                  instruction_data,
7028                                                                  n_instructions);
7029
7030         /* DMA many. */
7031         n_instructions = instr_pattern_dma_many_optimize(instructions,
7032                                                          instruction_data,
7033                                                          n_instructions);
7034
7035         return n_instructions;
7036 }
7037
7038 static int
7039 instruction_config(struct rte_swx_pipeline *p,
7040                    struct action *a,
7041                    const char **instructions,
7042                    uint32_t n_instructions)
7043 {
7044         struct instruction *instr = NULL;
7045         struct instruction_data *data = NULL;
7046         int err = 0;
7047         uint32_t i;
7048
7049         CHECK(n_instructions, EINVAL);
7050         CHECK(instructions, EINVAL);
7051         for (i = 0; i < n_instructions; i++)
7052                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7053
7054         /* Memory allocation. */
7055         instr = calloc(n_instructions, sizeof(struct instruction));
7056         if (!instr) {
7057                 err = -ENOMEM;
7058                 goto error;
7059         }
7060
7061         data = calloc(n_instructions, sizeof(struct instruction_data));
7062         if (!data) {
7063                 err = -ENOMEM;
7064                 goto error;
7065         }
7066
7067         for (i = 0; i < n_instructions; i++) {
7068                 char *string = strdup(instructions[i]);
7069                 if (!string) {
7070                         err = -ENOMEM;
7071                         goto error;
7072                 }
7073
7074                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7075                 if (err) {
7076                         free(string);
7077                         goto error;
7078                 }
7079
7080                 free(string);
7081         }
7082
7083         err = instr_label_check(data, n_instructions);
7084         if (err)
7085                 goto error;
7086
7087         err = instr_verify(p, a, instr, data, n_instructions);
7088         if (err)
7089                 goto error;
7090
7091         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7092
7093         err = instr_jmp_resolve(instr, data, n_instructions);
7094         if (err)
7095                 goto error;
7096
7097         if (a) {
7098                 a->instructions = instr;
7099                 a->instruction_data = data;
7100                 a->n_instructions = n_instructions;
7101         } else {
7102                 p->instructions = instr;
7103                 p->instruction_data = data;
7104                 p->n_instructions = n_instructions;
7105         }
7106
7107         return 0;
7108
7109 error:
7110         free(data);
7111         free(instr);
7112         return err;
7113 }
7114
7115 static instr_exec_t instruction_table[] = {
7116         [INSTR_RX] = instr_rx_exec,
7117         [INSTR_TX] = instr_tx_exec,
7118         [INSTR_TX_I] = instr_tx_i_exec,
7119         [INSTR_DROP] = instr_drop_exec,
7120         [INSTR_MIRROR] = instr_mirror_exec,
7121         [INSTR_RECIRCULATE] = instr_recirculate_exec,
7122         [INSTR_RECIRCID] = instr_recircid_exec,
7123
7124         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7125         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7126         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7127         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7128         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7129         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7130         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7131         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7132         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7133         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7134
7135         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7136         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7137         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7138         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7139         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7140         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7141         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7142         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7143         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7144
7145         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7146         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7147
7148         [INSTR_MOV] = instr_mov_exec,
7149         [INSTR_MOV_MH] = instr_mov_mh_exec,
7150         [INSTR_MOV_HM] = instr_mov_hm_exec,
7151         [INSTR_MOV_HH] = instr_mov_hh_exec,
7152         [INSTR_MOV_I] = instr_mov_i_exec,
7153
7154         [INSTR_DMA_HT] = instr_dma_ht_exec,
7155         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7156         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7157         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7158         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7159         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7160         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7161         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7162
7163         [INSTR_ALU_ADD] = instr_alu_add_exec,
7164         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7165         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7166         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7167         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7168         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7169
7170         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7171         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7172         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7173         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7174         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7175         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7176
7177         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7178         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7179         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7180         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7181
7182         [INSTR_ALU_AND] = instr_alu_and_exec,
7183         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7184         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7185         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7186         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7187
7188         [INSTR_ALU_OR] = instr_alu_or_exec,
7189         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7190         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7191         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7192         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7193
7194         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7195         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7196         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7197         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7198         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7199
7200         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7201         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7202         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7203         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7204         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7205         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7206
7207         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7208         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7209         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7210         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7211         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7212         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7213
7214         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7215         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7216         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7217
7218         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7219         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7220         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7221         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7222         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7223         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7224
7225         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7226         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7227         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7228         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7229         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7230         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7231         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7232         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7233         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7234
7235         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7236         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7237         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7238         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7239         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7240         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7241         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7242         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7243         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7244
7245         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7246         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7247         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7248
7249         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7250         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7251         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7252         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7253         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7254         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7255         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7256         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7257         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7258         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7259         [INSTR_METER_IMM] = instr_meter_imm_exec,
7260         [INSTR_METER_IMI] = instr_meter_imi_exec,
7261
7262         [INSTR_TABLE] = instr_table_exec,
7263         [INSTR_TABLE_AF] = instr_table_af_exec,
7264         [INSTR_SELECTOR] = instr_selector_exec,
7265         [INSTR_LEARNER] = instr_learner_exec,
7266         [INSTR_LEARNER_AF] = instr_learner_af_exec,
7267         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7268         [INSTR_LEARNER_REARM] = instr_rearm_exec,
7269         [INSTR_LEARNER_REARM_NEW] = instr_rearm_new_exec,
7270         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7271         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7272         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7273         [INSTR_HASH_FUNC] = instr_hash_func_exec,
7274
7275         [INSTR_JMP] = instr_jmp_exec,
7276         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7277         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7278         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7279         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7280         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7281         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7282
7283         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7284         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7285         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7286         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7287         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7288
7289         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7290         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7291         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7292         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7293         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7294
7295         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7296         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7297         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7298         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7299         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7300         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7301
7302         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7303         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7304         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7305         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7306         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7307         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7308
7309         [INSTR_RETURN] = instr_return_exec,
7310 };
7311
7312 static int
7313 instruction_table_build(struct rte_swx_pipeline *p)
7314 {
7315         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
7316                                       sizeof(struct instr_exec_t *));
7317         if (!p->instruction_table)
7318                 return -EINVAL;
7319
7320         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
7321
7322         return 0;
7323 }
7324
7325 static void
7326 instruction_table_build_free(struct rte_swx_pipeline *p)
7327 {
7328         if (!p->instruction_table)
7329                 return;
7330
7331         free(p->instruction_table);
7332         p->instruction_table = NULL;
7333 }
7334
7335 static void
7336 instruction_table_free(struct rte_swx_pipeline *p)
7337 {
7338         instruction_table_build_free(p);
7339 }
7340
7341 static inline void
7342 instr_exec(struct rte_swx_pipeline *p)
7343 {
7344         struct thread *t = &p->threads[p->thread_id];
7345         struct instruction *ip = t->ip;
7346         instr_exec_t instr = p->instruction_table[ip->type];
7347
7348         instr(p);
7349 }
7350
7351 /*
7352  * Action.
7353  */
7354 static struct action *
7355 action_find(struct rte_swx_pipeline *p, const char *name)
7356 {
7357         struct action *elem;
7358
7359         if (!name)
7360                 return NULL;
7361
7362         TAILQ_FOREACH(elem, &p->actions, node)
7363                 if (strcmp(elem->name, name) == 0)
7364                         return elem;
7365
7366         return NULL;
7367 }
7368
7369 static struct action *
7370 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7371 {
7372         struct action *action = NULL;
7373
7374         TAILQ_FOREACH(action, &p->actions, node)
7375                 if (action->id == id)
7376                         return action;
7377
7378         return NULL;
7379 }
7380
7381 static struct field *
7382 action_field_find(struct action *a, const char *name)
7383 {
7384         return a->st ? struct_type_field_find(a->st, name) : NULL;
7385 }
7386
7387 static struct field *
7388 action_field_parse(struct action *action, const char *name)
7389 {
7390         if (name[0] != 't' || name[1] != '.')
7391                 return NULL;
7392
7393         return action_field_find(action, &name[2]);
7394 }
7395
7396 static int
7397 action_has_nbo_args(struct action *a)
7398 {
7399         uint32_t i;
7400
7401         /* Return if the action does not have any args. */
7402         if (!a->st)
7403                 return 0; /* FALSE */
7404
7405         for (i = 0; i < a->st->n_fields; i++)
7406                 if (a->args_endianness[i])
7407                         return 1; /* TRUE */
7408
7409         return 0; /* FALSE */
7410 }
7411
7412 static int
7413 action_does_learning(struct action *a)
7414 {
7415         uint32_t i;
7416
7417         for (i = 0; i < a->n_instructions; i++)
7418                 switch (a->instructions[i].type) {
7419                 case INSTR_LEARNER_LEARN:
7420                         return 1; /* TRUE */
7421
7422                 case INSTR_LEARNER_FORGET:
7423                         return 1; /* TRUE */
7424
7425                 default:
7426                         continue;
7427                 }
7428
7429         return 0; /* FALSE */
7430 }
7431
7432 int
7433 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7434                                const char *name,
7435                                const char *args_struct_type_name,
7436                                const char **instructions,
7437                                uint32_t n_instructions)
7438 {
7439         struct struct_type *args_struct_type = NULL;
7440         struct action *a = NULL;
7441         int status = 0;
7442
7443         CHECK(p, EINVAL);
7444
7445         CHECK_NAME(name, EINVAL);
7446         CHECK(!action_find(p, name), EEXIST);
7447
7448         if (args_struct_type_name) {
7449                 CHECK_NAME(args_struct_type_name, EINVAL);
7450                 args_struct_type = struct_type_find(p, args_struct_type_name);
7451                 CHECK(args_struct_type, EINVAL);
7452                 CHECK(!args_struct_type->var_size, EINVAL);
7453         }
7454
7455         /* Node allocation. */
7456         a = calloc(1, sizeof(struct action));
7457         if (!a) {
7458                 status = -ENOMEM;
7459                 goto error;
7460         }
7461
7462         if (args_struct_type) {
7463                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7464                 if (!a->args_endianness) {
7465                         status = -ENOMEM;
7466                         goto error;
7467                 }
7468         }
7469
7470         /* Node initialization. */
7471         strcpy(a->name, name);
7472         a->st = args_struct_type;
7473         a->id = p->n_actions;
7474
7475         /* Instruction translation. */
7476         status = instruction_config(p, a, instructions, n_instructions);
7477         if (status)
7478                 goto error;
7479
7480         /* Node add to tailq. */
7481         TAILQ_INSERT_TAIL(&p->actions, a, node);
7482         p->n_actions++;
7483
7484         return 0;
7485
7486 error:
7487         if (!a)
7488                 return status;
7489
7490         free(a->args_endianness);
7491         free(a->instructions);
7492         free(a->instruction_data);
7493         free(a);
7494
7495         return status;
7496 }
7497
7498 static int
7499 action_build(struct rte_swx_pipeline *p)
7500 {
7501         struct action *action;
7502
7503         /* p->action_instructions. */
7504         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7505         CHECK(p->action_instructions, ENOMEM);
7506
7507         TAILQ_FOREACH(action, &p->actions, node)
7508                 p->action_instructions[action->id] = action->instructions;
7509
7510         /* p->action_funcs. */
7511         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7512         CHECK(p->action_funcs, ENOMEM);
7513
7514         return 0;
7515 }
7516
7517 static void
7518 action_build_free(struct rte_swx_pipeline *p)
7519 {
7520         free(p->action_funcs);
7521         p->action_funcs = NULL;
7522
7523         free(p->action_instructions);
7524         p->action_instructions = NULL;
7525 }
7526
7527 static void
7528 action_free(struct rte_swx_pipeline *p)
7529 {
7530         action_build_free(p);
7531
7532         for ( ; ; ) {
7533                 struct action *action;
7534
7535                 action = TAILQ_FIRST(&p->actions);
7536                 if (!action)
7537                         break;
7538
7539                 TAILQ_REMOVE(&p->actions, action, node);
7540                 free(action->args_endianness);
7541                 free(action->instructions);
7542                 free(action->instruction_data);
7543                 free(action);
7544         }
7545 }
7546
7547 static uint32_t
7548 action_arg_src_mov_count(struct action *a,
7549                          uint32_t arg_id,
7550                          struct instruction *instructions,
7551                          struct instruction_data *instruction_data,
7552                          uint32_t n_instructions)
7553 {
7554         uint32_t offset, n_users = 0, i;
7555
7556         if (!a->st ||
7557             (arg_id >= a->st->n_fields) ||
7558             !instructions ||
7559             !instruction_data ||
7560             !n_instructions)
7561                 return 0;
7562
7563         offset = a->st->fields[arg_id].offset / 8;
7564
7565         for (i = 0; i < n_instructions; i++) {
7566                 struct instruction *instr = &instructions[i];
7567                 struct instruction_data *data = &instruction_data[i];
7568
7569                 if (data->invalid ||
7570                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7571                     instr->mov.src.struct_id ||
7572                     (instr->mov.src.offset != offset))
7573                         continue;
7574
7575                 n_users++;
7576         }
7577
7578         return n_users;
7579 }
7580
7581 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7582 #define field_ntoh(val, n_bits) (ntoh64((val) << (64 - n_bits)))
7583 #define field_hton(val, n_bits) (hton64((val) << (64 - n_bits)))
7584 #else
7585 #define field_ntoh(val, n_bits) (val)
7586 #define field_hton(val, n_bits) (val)
7587 #endif
7588
7589 #define ACTION_ARGS_TOKENS_MAX 256
7590
7591 static int
7592 action_args_parse(struct action *a, const char *args, uint8_t *data)
7593 {
7594         char *tokens[ACTION_ARGS_TOKENS_MAX], *s0 = NULL, *s;
7595         uint32_t n_tokens = 0, offset = 0, i;
7596         int status = 0;
7597
7598         /* Checks. */
7599         if (!a->st || !args || !args[0]) {
7600                 status = -EINVAL;
7601                 goto error;
7602         }
7603
7604         /* Memory allocation. */
7605         s0 = strdup(args);
7606         if (!s0) {
7607                 status = -ENOMEM;
7608                 goto error;
7609         }
7610
7611         /* Parse the string into tokens. */
7612         for (s = s0; ; ) {
7613                 char *token;
7614
7615                 token = strtok_r(s, " \f\n\r\t\v", &s);
7616                 if (!token)
7617                         break;
7618
7619                 if (n_tokens >= RTE_DIM(tokens)) {
7620                         status = -EINVAL;
7621                         goto error;
7622                 }
7623
7624                 tokens[n_tokens] = token;
7625                 n_tokens++;
7626         }
7627
7628         /* More checks. */
7629         if (n_tokens != a->st->n_fields * 2) {
7630                 status = -EINVAL;
7631                 goto error;
7632         }
7633
7634         /* Process the action arguments. */
7635         for (i = 0; i < a->st->n_fields; i++) {
7636                 struct field *f = &a->st->fields[i];
7637                 char *arg_name = tokens[i * 2];
7638                 char *arg_val = tokens[i * 2 + 1];
7639                 uint64_t val;
7640
7641                 if (strcmp(arg_name, f->name)) {
7642                         status = -EINVAL;
7643                         goto error;
7644                 }
7645
7646                 val = strtoull(arg_val, &arg_val, 0);
7647                 if (arg_val[0]) {
7648                         status = -EINVAL;
7649                         goto error;
7650                 }
7651
7652                 /* Endianness conversion. */
7653                 if (a->args_endianness[i])
7654                         val = field_hton(val, f->n_bits);
7655
7656                 /* Copy to entry. */
7657                 memcpy(&data[offset], (uint8_t *)&val, f->n_bits / 8);
7658                 offset += f->n_bits / 8;
7659         }
7660
7661 error:
7662         free(s0);
7663         return status;
7664 }
7665
7666 /*
7667  * Table.
7668  */
7669 static struct table_type *
7670 table_type_find(struct rte_swx_pipeline *p, const char *name)
7671 {
7672         struct table_type *elem;
7673
7674         TAILQ_FOREACH(elem, &p->table_types, node)
7675                 if (strcmp(elem->name, name) == 0)
7676                         return elem;
7677
7678         return NULL;
7679 }
7680
7681 static struct table_type *
7682 table_type_resolve(struct rte_swx_pipeline *p,
7683                    const char *recommended_type_name,
7684                    enum rte_swx_table_match_type match_type)
7685 {
7686         struct table_type *elem;
7687
7688         /* Only consider the recommended type if the match type is correct. */
7689         if (recommended_type_name)
7690                 TAILQ_FOREACH(elem, &p->table_types, node)
7691                         if (!strcmp(elem->name, recommended_type_name) &&
7692                             (elem->match_type == match_type))
7693                                 return elem;
7694
7695         /* Ignore the recommended type and get the first element with this match
7696          * type.
7697          */
7698         TAILQ_FOREACH(elem, &p->table_types, node)
7699                 if (elem->match_type == match_type)
7700                         return elem;
7701
7702         return NULL;
7703 }
7704
7705 static struct table *
7706 table_find(struct rte_swx_pipeline *p, const char *name)
7707 {
7708         struct table *elem;
7709
7710         TAILQ_FOREACH(elem, &p->tables, node)
7711                 if (strcmp(elem->name, name) == 0)
7712                         return elem;
7713
7714         return NULL;
7715 }
7716
7717 static struct table *
7718 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7719 {
7720         struct table *table = NULL;
7721
7722         TAILQ_FOREACH(table, &p->tables, node)
7723                 if (table->id == id)
7724                         return table;
7725
7726         return NULL;
7727 }
7728
7729 int
7730 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7731                                      const char *name,
7732                                      enum rte_swx_table_match_type match_type,
7733                                      struct rte_swx_table_ops *ops)
7734 {
7735         struct table_type *elem;
7736
7737         CHECK(p, EINVAL);
7738
7739         CHECK_NAME(name, EINVAL);
7740         CHECK(!table_type_find(p, name), EEXIST);
7741
7742         CHECK(ops, EINVAL);
7743         CHECK(ops->create, EINVAL);
7744         CHECK(ops->lkp, EINVAL);
7745         CHECK(ops->free, EINVAL);
7746
7747         /* Node allocation. */
7748         elem = calloc(1, sizeof(struct table_type));
7749         CHECK(elem, ENOMEM);
7750
7751         /* Node initialization. */
7752         strcpy(elem->name, name);
7753         elem->match_type = match_type;
7754         memcpy(&elem->ops, ops, sizeof(*ops));
7755
7756         /* Node add to tailq. */
7757         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7758
7759         return 0;
7760 }
7761
7762 static int
7763 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7764                          uint32_t n_fields,
7765                          enum rte_swx_table_match_type *match_type)
7766 {
7767         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7768
7769         for (i = 0; i < n_fields; i++) {
7770                 struct rte_swx_match_field_params  *f = &fields[i];
7771
7772                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7773                         n_fields_em++;
7774
7775                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7776                         n_fields_lpm++;
7777         }
7778
7779         if ((n_fields_lpm > 1) ||
7780             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7781                 return -EINVAL;
7782
7783         *match_type = (n_fields_em == n_fields) ?
7784                        RTE_SWX_TABLE_MATCH_EXACT :
7785                        RTE_SWX_TABLE_MATCH_WILDCARD;
7786
7787         return 0;
7788 }
7789
7790 static int
7791 table_match_fields_check(struct rte_swx_pipeline *p,
7792                          struct rte_swx_pipeline_table_params *params,
7793                          struct header **header)
7794 {
7795         struct header *h0 = NULL;
7796         struct field *hf, *mf;
7797         uint32_t *offset = NULL, i;
7798         int status = 0;
7799
7800         /* Return if no match fields. */
7801         if (!params->n_fields) {
7802                 if (params->fields) {
7803                         status = -EINVAL;
7804                         goto end;
7805                 }
7806
7807                 if (header)
7808                         *header = NULL;
7809
7810                 return 0;
7811         }
7812
7813         /* Memory allocation. */
7814         offset = calloc(params->n_fields, sizeof(uint32_t));
7815         if (!offset) {
7816                 status = -ENOMEM;
7817                 goto end;
7818         }
7819
7820         /* Check that all the match fields belong to either the same header or
7821          * to the meta-data.
7822          */
7823         hf = header_field_parse(p, params->fields[0].name, &h0);
7824         mf = metadata_field_parse(p, params->fields[0].name);
7825         if ((!hf && !mf) || (hf && hf->var_size)) {
7826                 status = -EINVAL;
7827                 goto end;
7828         }
7829
7830         offset[0] = h0 ? hf->offset : mf->offset;
7831
7832         for (i = 1; i < params->n_fields; i++)
7833                 if (h0) {
7834                         struct header *h;
7835
7836                         hf = header_field_parse(p, params->fields[i].name, &h);
7837                         if (!hf || (h->id != h0->id) || hf->var_size) {
7838                                 status = -EINVAL;
7839                                 goto end;
7840                         }
7841
7842                         offset[i] = hf->offset;
7843                 } else {
7844                         mf = metadata_field_parse(p, params->fields[i].name);
7845                         if (!mf) {
7846                                 status = -EINVAL;
7847                                 goto end;
7848                         }
7849
7850                         offset[i] = mf->offset;
7851                 }
7852
7853         /* Check that there are no duplicated match fields. */
7854         for (i = 0; i < params->n_fields; i++) {
7855                 uint32_t j;
7856
7857                 for (j = 0; j < i; j++)
7858                         if (offset[j] == offset[i]) {
7859                                 status = -EINVAL;
7860                                 goto end;
7861                         }
7862         }
7863
7864         /* Return. */
7865         if (header)
7866                 *header = h0;
7867
7868 end:
7869         free(offset);
7870         return status;
7871 }
7872
7873 int
7874 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7875                               const char *name,
7876                               struct rte_swx_pipeline_table_params *params,
7877                               const char *recommended_table_type_name,
7878                               const char *args,
7879                               uint32_t size)
7880 {
7881         struct table_type *type;
7882         struct table *t = NULL;
7883         struct action *default_action;
7884         struct header *header = NULL;
7885         uint32_t action_data_size_max = 0, i;
7886         int status = 0;
7887
7888         CHECK(p, EINVAL);
7889
7890         CHECK_NAME(name, EINVAL);
7891         CHECK(!table_find(p, name), EEXIST);
7892         CHECK(!selector_find(p, name), EEXIST);
7893         CHECK(!learner_find(p, name), EEXIST);
7894
7895         CHECK(params, EINVAL);
7896
7897         /* Match checks. */
7898         status = table_match_fields_check(p, params, &header);
7899         if (status)
7900                 return status;
7901
7902         /* Action checks. */
7903         CHECK(params->n_actions, EINVAL);
7904         CHECK(params->action_names, EINVAL);
7905         for (i = 0; i < params->n_actions; i++) {
7906                 const char *action_name = params->action_names[i];
7907                 struct action *a;
7908                 uint32_t action_data_size;
7909                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7910
7911                 CHECK_NAME(action_name, EINVAL);
7912
7913                 a = action_find(p, action_name);
7914                 CHECK(a, EINVAL);
7915                 CHECK(!action_does_learning(a), EINVAL);
7916
7917                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7918                 if (action_data_size > action_data_size_max)
7919                         action_data_size_max = action_data_size;
7920
7921                 if (params->action_is_for_table_entries)
7922                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7923                 if (params->action_is_for_default_entry)
7924                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7925                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7926         }
7927
7928         CHECK_NAME(params->default_action_name, EINVAL);
7929         for (i = 0; i < p->n_actions; i++)
7930                 if (!strcmp(params->action_names[i],
7931                             params->default_action_name))
7932                         break;
7933         CHECK(i < params->n_actions, EINVAL);
7934         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7935               EINVAL);
7936
7937         default_action = action_find(p, params->default_action_name);
7938         CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
7939               EINVAL);
7940
7941         /* Table type checks. */
7942         if (recommended_table_type_name)
7943                 CHECK_NAME(recommended_table_type_name, EINVAL);
7944
7945         if (params->n_fields) {
7946                 enum rte_swx_table_match_type match_type;
7947
7948                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7949                 if (status)
7950                         return status;
7951
7952                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7953                 CHECK(type, EINVAL);
7954         } else {
7955                 type = NULL;
7956         }
7957
7958         /* Memory allocation. */
7959         t = calloc(1, sizeof(struct table));
7960         if (!t) {
7961                 status = -ENOMEM;
7962                 goto error;
7963         }
7964
7965         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7966         if (!t->fields) {
7967                 status = -ENOMEM;
7968                 goto error;
7969         }
7970
7971         t->actions = calloc(params->n_actions, sizeof(struct action *));
7972         if (!t->actions) {
7973                 status = -ENOMEM;
7974                 goto error;
7975         }
7976
7977         if (action_data_size_max) {
7978                 t->default_action_data = calloc(1, action_data_size_max);
7979                 if (!t->default_action_data) {
7980                         status = -ENOMEM;
7981                         goto error;
7982                 }
7983         }
7984
7985         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7986         if (!t->action_is_for_table_entries) {
7987                 status = -ENOMEM;
7988                 goto error;
7989         }
7990
7991         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7992         if (!t->action_is_for_default_entry) {
7993                 status = -ENOMEM;
7994                 goto error;
7995         }
7996
7997         /* Node initialization. */
7998         strcpy(t->name, name);
7999         if (args && args[0])
8000                 strcpy(t->args, args);
8001         t->type = type;
8002
8003         for (i = 0; i < params->n_fields; i++) {
8004                 struct rte_swx_match_field_params *field = &params->fields[i];
8005                 struct match_field *f = &t->fields[i];
8006
8007                 f->match_type = field->match_type;
8008                 f->field = header ?
8009                         header_field_parse(p, field->name, NULL) :
8010                         metadata_field_parse(p, field->name);
8011         }
8012         t->n_fields = params->n_fields;
8013         t->header = header;
8014
8015         for (i = 0; i < params->n_actions; i++) {
8016                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8017
8018                 if (params->action_is_for_table_entries)
8019                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8020                 if (params->action_is_for_default_entry)
8021                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8022
8023                 t->actions[i] = action_find(p, params->action_names[i]);
8024                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
8025                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
8026         }
8027         t->default_action = default_action;
8028         if (default_action->st) {
8029                 status = action_args_parse(default_action,
8030                                            params->default_action_args,
8031                                            t->default_action_data);
8032                 if (status)
8033                         goto error;
8034         }
8035
8036         t->n_actions = params->n_actions;
8037         t->default_action_is_const = params->default_action_is_const;
8038         t->action_data_size_max = action_data_size_max;
8039
8040         t->size = size;
8041         t->id = p->n_tables;
8042
8043         /* Node add to tailq. */
8044         TAILQ_INSERT_TAIL(&p->tables, t, node);
8045         p->n_tables++;
8046
8047         return 0;
8048
8049 error:
8050         if (!t)
8051                 return status;
8052
8053         free(t->action_is_for_default_entry);
8054         free(t->action_is_for_table_entries);
8055         free(t->default_action_data);
8056         free(t->actions);
8057         free(t->fields);
8058         free(t);
8059
8060         return status;
8061 }
8062
8063 static struct rte_swx_table_params *
8064 table_params_get(struct table *table)
8065 {
8066         struct rte_swx_table_params *params;
8067         struct field *first, *last;
8068         uint8_t *key_mask;
8069         uint32_t key_size, key_offset, action_data_size, i;
8070
8071         /* Memory allocation. */
8072         params = calloc(1, sizeof(struct rte_swx_table_params));
8073         if (!params)
8074                 return NULL;
8075
8076         /* Find first (smallest offset) and last (biggest offset) match fields. */
8077         first = table->fields[0].field;
8078         last = table->fields[0].field;
8079
8080         for (i = 0; i < table->n_fields; i++) {
8081                 struct field *f = table->fields[i].field;
8082
8083                 if (f->offset < first->offset)
8084                         first = f;
8085
8086                 if (f->offset > last->offset)
8087                         last = f;
8088         }
8089
8090         /* Key offset and size. */
8091         key_offset = first->offset / 8;
8092         key_size = (last->offset + last->n_bits - first->offset) / 8;
8093
8094         /* Memory allocation. */
8095         key_mask = calloc(1, key_size);
8096         if (!key_mask) {
8097                 free(params);
8098                 return NULL;
8099         }
8100
8101         /* Key mask. */
8102         for (i = 0; i < table->n_fields; i++) {
8103                 struct field *f = table->fields[i].field;
8104                 uint32_t start = (f->offset - first->offset) / 8;
8105                 size_t size = f->n_bits / 8;
8106
8107                 memset(&key_mask[start], 0xFF, size);
8108         }
8109
8110         /* Action data size. */
8111         action_data_size = 0;
8112         for (i = 0; i < table->n_actions; i++) {
8113                 struct action *action = table->actions[i];
8114                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8115
8116                 if (ads > action_data_size)
8117                         action_data_size = ads;
8118         }
8119
8120         /* Fill in. */
8121         params->match_type = table->type->match_type;
8122         params->key_size = key_size;
8123         params->key_offset = key_offset;
8124         params->key_mask0 = key_mask;
8125         params->action_data_size = action_data_size;
8126         params->n_keys_max = table->size;
8127
8128         return params;
8129 }
8130
8131 static void
8132 table_params_free(struct rte_swx_table_params *params)
8133 {
8134         if (!params)
8135                 return;
8136
8137         free(params->key_mask0);
8138         free(params);
8139 }
8140
8141 static int
8142 table_stub_lkp(void *table __rte_unused,
8143                void *mailbox __rte_unused,
8144                uint8_t **key __rte_unused,
8145                uint64_t *action_id __rte_unused,
8146                uint8_t **action_data __rte_unused,
8147                int *hit)
8148 {
8149         *hit = 0;
8150         return 1; /* DONE. */
8151 }
8152
8153 static int
8154 table_build(struct rte_swx_pipeline *p)
8155 {
8156         uint32_t i;
8157
8158         /* Per pipeline: table statistics. */
8159         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8160         CHECK(p->table_stats, ENOMEM);
8161
8162         for (i = 0; i < p->n_tables; i++) {
8163                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8164                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8165         }
8166
8167         /* Per thread: table runt-time. */
8168         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8169                 struct thread *t = &p->threads[i];
8170                 struct table *table;
8171
8172                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8173                 CHECK(t->tables, ENOMEM);
8174
8175                 TAILQ_FOREACH(table, &p->tables, node) {
8176                         struct table_runtime *r = &t->tables[table->id];
8177
8178                         if (table->type) {
8179                                 uint64_t size;
8180
8181                                 size = table->type->ops.mailbox_size_get();
8182
8183                                 /* r->func. */
8184                                 r->func = table->type->ops.lkp;
8185
8186                                 /* r->mailbox. */
8187                                 if (size) {
8188                                         r->mailbox = calloc(1, size);
8189                                         CHECK(r->mailbox, ENOMEM);
8190                                 }
8191
8192                                 /* r->key. */
8193                                 r->key = table->header ?
8194                                         &t->structs[table->header->struct_id] :
8195                                         &t->structs[p->metadata_struct_id];
8196                         } else {
8197                                 r->func = table_stub_lkp;
8198                         }
8199                 }
8200         }
8201
8202         return 0;
8203 }
8204
8205 static void
8206 table_build_free(struct rte_swx_pipeline *p)
8207 {
8208         uint32_t i;
8209
8210         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8211                 struct thread *t = &p->threads[i];
8212                 uint32_t j;
8213
8214                 if (!t->tables)
8215                         continue;
8216
8217                 for (j = 0; j < p->n_tables; j++) {
8218                         struct table_runtime *r = &t->tables[j];
8219
8220                         free(r->mailbox);
8221                 }
8222
8223                 free(t->tables);
8224                 t->tables = NULL;
8225         }
8226
8227         if (p->table_stats) {
8228                 for (i = 0; i < p->n_tables; i++)
8229                         free(p->table_stats[i].n_pkts_action);
8230
8231                 free(p->table_stats);
8232         }
8233 }
8234
8235 static void
8236 table_free(struct rte_swx_pipeline *p)
8237 {
8238         table_build_free(p);
8239
8240         /* Tables. */
8241         for ( ; ; ) {
8242                 struct table *elem;
8243
8244                 elem = TAILQ_FIRST(&p->tables);
8245                 if (!elem)
8246                         break;
8247
8248                 TAILQ_REMOVE(&p->tables, elem, node);
8249                 free(elem->fields);
8250                 free(elem->actions);
8251                 free(elem->default_action_data);
8252                 free(elem);
8253         }
8254
8255         /* Table types. */
8256         for ( ; ; ) {
8257                 struct table_type *elem;
8258
8259                 elem = TAILQ_FIRST(&p->table_types);
8260                 if (!elem)
8261                         break;
8262
8263                 TAILQ_REMOVE(&p->table_types, elem, node);
8264                 free(elem);
8265         }
8266 }
8267
8268 /*
8269  * Selector.
8270  */
8271 static struct selector *
8272 selector_find(struct rte_swx_pipeline *p, const char *name)
8273 {
8274         struct selector *s;
8275
8276         TAILQ_FOREACH(s, &p->selectors, node)
8277                 if (strcmp(s->name, name) == 0)
8278                         return s;
8279
8280         return NULL;
8281 }
8282
8283 static struct selector *
8284 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8285 {
8286         struct selector *s = NULL;
8287
8288         TAILQ_FOREACH(s, &p->selectors, node)
8289                 if (s->id == id)
8290                         return s;
8291
8292         return NULL;
8293 }
8294
8295 static int
8296 selector_fields_check(struct rte_swx_pipeline *p,
8297                       struct rte_swx_pipeline_selector_params *params,
8298                       struct header **header)
8299 {
8300         struct header *h0 = NULL;
8301         struct field *hf, *mf;
8302         uint32_t i;
8303
8304         /* Return if no selector fields. */
8305         if (!params->n_selector_fields || !params->selector_field_names)
8306                 return -EINVAL;
8307
8308         /* Check that all the selector fields either belong to the same header
8309          * or are all meta-data fields.
8310          */
8311         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8312         mf = metadata_field_parse(p, params->selector_field_names[0]);
8313         if (!hf && !mf)
8314                 return -EINVAL;
8315
8316         for (i = 1; i < params->n_selector_fields; i++)
8317                 if (h0) {
8318                         struct header *h;
8319
8320                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8321                         if (!hf || (h->id != h0->id))
8322                                 return -EINVAL;
8323                 } else {
8324                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8325                         if (!mf)
8326                                 return -EINVAL;
8327                 }
8328
8329         /* Check that there are no duplicated match fields. */
8330         for (i = 0; i < params->n_selector_fields; i++) {
8331                 const char *field_name = params->selector_field_names[i];
8332                 uint32_t j;
8333
8334                 for (j = i + 1; j < params->n_selector_fields; j++)
8335                         if (!strcmp(params->selector_field_names[j], field_name))
8336                                 return -EINVAL;
8337         }
8338
8339         /* Return. */
8340         if (header)
8341                 *header = h0;
8342
8343         return 0;
8344 }
8345
8346 int
8347 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8348                                  const char *name,
8349                                  struct rte_swx_pipeline_selector_params *params)
8350 {
8351         struct selector *s;
8352         struct header *selector_header = NULL;
8353         struct field *group_id_field, *member_id_field;
8354         uint32_t i;
8355         int status = 0;
8356
8357         CHECK(p, EINVAL);
8358
8359         CHECK_NAME(name, EINVAL);
8360         CHECK(!table_find(p, name), EEXIST);
8361         CHECK(!selector_find(p, name), EEXIST);
8362         CHECK(!learner_find(p, name), EEXIST);
8363
8364         CHECK(params, EINVAL);
8365
8366         CHECK_NAME(params->group_id_field_name, EINVAL);
8367         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8368         CHECK(group_id_field, EINVAL);
8369
8370         for (i = 0; i < params->n_selector_fields; i++) {
8371                 const char *field_name = params->selector_field_names[i];
8372
8373                 CHECK_NAME(field_name, EINVAL);
8374         }
8375         status = selector_fields_check(p, params, &selector_header);
8376         if (status)
8377                 return status;
8378
8379         CHECK_NAME(params->member_id_field_name, EINVAL);
8380         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8381         CHECK(member_id_field, EINVAL);
8382
8383         CHECK(params->n_groups_max, EINVAL);
8384
8385         CHECK(params->n_members_per_group_max, EINVAL);
8386
8387         /* Memory allocation. */
8388         s = calloc(1, sizeof(struct selector));
8389         if (!s) {
8390                 status = -ENOMEM;
8391                 goto error;
8392         }
8393
8394         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8395         if (!s->selector_fields) {
8396                 status = -ENOMEM;
8397                 goto error;
8398         }
8399
8400         /* Node initialization. */
8401         strcpy(s->name, name);
8402
8403         s->group_id_field = group_id_field;
8404
8405         for (i = 0; i < params->n_selector_fields; i++) {
8406                 const char *field_name = params->selector_field_names[i];
8407
8408                 s->selector_fields[i] = selector_header ?
8409                         header_field_parse(p, field_name, NULL) :
8410                         metadata_field_parse(p, field_name);
8411         }
8412
8413         s->n_selector_fields = params->n_selector_fields;
8414
8415         s->selector_header = selector_header;
8416
8417         s->member_id_field = member_id_field;
8418
8419         s->n_groups_max = params->n_groups_max;
8420
8421         s->n_members_per_group_max = params->n_members_per_group_max;
8422
8423         s->id = p->n_selectors;
8424
8425         /* Node add to tailq. */
8426         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8427         p->n_selectors++;
8428
8429         return 0;
8430
8431 error:
8432         if (!s)
8433                 return status;
8434
8435         free(s->selector_fields);
8436
8437         free(s);
8438
8439         return status;
8440 }
8441
8442 static void
8443 selector_params_free(struct rte_swx_table_selector_params *params)
8444 {
8445         if (!params)
8446                 return;
8447
8448         free(params->selector_mask);
8449
8450         free(params);
8451 }
8452
8453 static struct rte_swx_table_selector_params *
8454 selector_table_params_get(struct selector *s)
8455 {
8456         struct rte_swx_table_selector_params *params = NULL;
8457         struct field *first, *last;
8458         uint32_t i;
8459
8460         /* Memory allocation. */
8461         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8462         if (!params)
8463                 goto error;
8464
8465         /* Group ID. */
8466         params->group_id_offset = s->group_id_field->offset / 8;
8467
8468         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8469         first = s->selector_fields[0];
8470         last = s->selector_fields[0];
8471
8472         for (i = 0; i < s->n_selector_fields; i++) {
8473                 struct field *f = s->selector_fields[i];
8474
8475                 if (f->offset < first->offset)
8476                         first = f;
8477
8478                 if (f->offset > last->offset)
8479                         last = f;
8480         }
8481
8482         /* Selector offset and size. */
8483         params->selector_offset = first->offset / 8;
8484         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8485
8486         /* Memory allocation. */
8487         params->selector_mask = calloc(1, params->selector_size);
8488         if (!params->selector_mask)
8489                 goto error;
8490
8491         /* Selector mask. */
8492         for (i = 0; i < s->n_selector_fields; i++) {
8493                 struct field *f = s->selector_fields[i];
8494                 uint32_t start = (f->offset - first->offset) / 8;
8495                 size_t size = f->n_bits / 8;
8496
8497                 memset(&params->selector_mask[start], 0xFF, size);
8498         }
8499
8500         /* Member ID. */
8501         params->member_id_offset = s->member_id_field->offset / 8;
8502
8503         /* Maximum number of groups. */
8504         params->n_groups_max = s->n_groups_max;
8505
8506         /* Maximum number of members per group. */
8507         params->n_members_per_group_max = s->n_members_per_group_max;
8508
8509         return params;
8510
8511 error:
8512         selector_params_free(params);
8513         return NULL;
8514 }
8515
8516 static void
8517 selector_build_free(struct rte_swx_pipeline *p)
8518 {
8519         uint32_t i;
8520
8521         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8522                 struct thread *t = &p->threads[i];
8523                 uint32_t j;
8524
8525                 if (!t->selectors)
8526                         continue;
8527
8528                 for (j = 0; j < p->n_selectors; j++) {
8529                         struct selector_runtime *r = &t->selectors[j];
8530
8531                         free(r->mailbox);
8532                 }
8533
8534                 free(t->selectors);
8535                 t->selectors = NULL;
8536         }
8537
8538         free(p->selector_stats);
8539         p->selector_stats = NULL;
8540 }
8541
8542 static int
8543 selector_build(struct rte_swx_pipeline *p)
8544 {
8545         uint32_t i;
8546         int status = 0;
8547
8548         /* Per pipeline: selector statistics. */
8549         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8550         if (!p->selector_stats) {
8551                 status = -ENOMEM;
8552                 goto error;
8553         }
8554
8555         /* Per thread: selector run-time. */
8556         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8557                 struct thread *t = &p->threads[i];
8558                 struct selector *s;
8559
8560                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8561                 if (!t->selectors) {
8562                         status = -ENOMEM;
8563                         goto error;
8564                 }
8565
8566                 TAILQ_FOREACH(s, &p->selectors, node) {
8567                         struct selector_runtime *r = &t->selectors[s->id];
8568                         uint64_t size;
8569
8570                         /* r->mailbox. */
8571                         size = rte_swx_table_selector_mailbox_size_get();
8572                         if (size) {
8573                                 r->mailbox = calloc(1, size);
8574                                 if (!r->mailbox) {
8575                                         status = -ENOMEM;
8576                                         goto error;
8577                                 }
8578                         }
8579
8580                         /* r->group_id_buffer. */
8581                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8582
8583                         /* r->selector_buffer. */
8584                         r->selector_buffer = s->selector_header ?
8585                                 &t->structs[s->selector_header->struct_id] :
8586                                 &t->structs[p->metadata_struct_id];
8587
8588                         /* r->member_id_buffer. */
8589                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8590                 }
8591         }
8592
8593         return 0;
8594
8595 error:
8596         selector_build_free(p);
8597         return status;
8598 }
8599
8600 static void
8601 selector_free(struct rte_swx_pipeline *p)
8602 {
8603         selector_build_free(p);
8604
8605         /* Selector tables. */
8606         for ( ; ; ) {
8607                 struct selector *elem;
8608
8609                 elem = TAILQ_FIRST(&p->selectors);
8610                 if (!elem)
8611                         break;
8612
8613                 TAILQ_REMOVE(&p->selectors, elem, node);
8614                 free(elem->selector_fields);
8615                 free(elem);
8616         }
8617 }
8618
8619 /*
8620  * Learner table.
8621  */
8622 static struct learner *
8623 learner_find(struct rte_swx_pipeline *p, const char *name)
8624 {
8625         struct learner *l;
8626
8627         TAILQ_FOREACH(l, &p->learners, node)
8628                 if (!strcmp(l->name, name))
8629                         return l;
8630
8631         return NULL;
8632 }
8633
8634 static struct learner *
8635 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8636 {
8637         struct learner *l = NULL;
8638
8639         TAILQ_FOREACH(l, &p->learners, node)
8640                 if (l->id == id)
8641                         return l;
8642
8643         return NULL;
8644 }
8645
8646 static int
8647 learner_match_fields_check(struct rte_swx_pipeline *p,
8648                            struct rte_swx_pipeline_learner_params *params,
8649                            struct header **header)
8650 {
8651         struct header *h0 = NULL;
8652         struct field *hf, *mf;
8653         uint32_t i;
8654
8655         /* Return if no match fields. */
8656         if (!params->n_fields || !params->field_names)
8657                 return -EINVAL;
8658
8659         /* Check that all the match fields either belong to the same header
8660          * or are all meta-data fields.
8661          */
8662         hf = header_field_parse(p, params->field_names[0], &h0);
8663         mf = metadata_field_parse(p, params->field_names[0]);
8664         if (!hf && !mf)
8665                 return -EINVAL;
8666
8667         for (i = 1; i < params->n_fields; i++)
8668                 if (h0) {
8669                         struct header *h;
8670
8671                         hf = header_field_parse(p, params->field_names[i], &h);
8672                         if (!hf || (h->id != h0->id))
8673                                 return -EINVAL;
8674                 } else {
8675                         mf = metadata_field_parse(p, params->field_names[i]);
8676                         if (!mf)
8677                                 return -EINVAL;
8678                 }
8679
8680         /* Check that there are no duplicated match fields. */
8681         for (i = 0; i < params->n_fields; i++) {
8682                 const char *field_name = params->field_names[i];
8683                 uint32_t j;
8684
8685                 for (j = i + 1; j < params->n_fields; j++)
8686                         if (!strcmp(params->field_names[j], field_name))
8687                                 return -EINVAL;
8688         }
8689
8690         /* Return. */
8691         if (header)
8692                 *header = h0;
8693
8694         return 0;
8695 }
8696
8697 static int
8698 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8699 {
8700         struct struct_type *mst = p->metadata_st, *ast = a->st;
8701         struct field *mf, *af;
8702         uint32_t mf_pos, i;
8703
8704         if (!ast) {
8705                 if (mf_name)
8706                         return -EINVAL;
8707
8708                 return 0;
8709         }
8710
8711         /* Check that mf_name is the name of a valid meta-data field. */
8712         CHECK_NAME(mf_name, EINVAL);
8713         mf = metadata_field_parse(p, mf_name);
8714         CHECK(mf, EINVAL);
8715
8716         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8717          * all the action arguments.
8718          */
8719         mf_pos = mf - mst->fields;
8720         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8721
8722         /* Check that the size of each of the identified meta-data fields matches exactly the size
8723          * of the corresponding action argument.
8724          */
8725         for (i = 0; i < ast->n_fields; i++) {
8726                 mf = &mst->fields[mf_pos + i];
8727                 af = &ast->fields[i];
8728
8729                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8730         }
8731
8732         return 0;
8733 }
8734
8735 static int
8736 learner_action_learning_check(struct rte_swx_pipeline *p,
8737                               struct action *action,
8738                               const char **action_names,
8739                               uint32_t n_actions)
8740 {
8741         uint32_t i;
8742
8743         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8744          * the action passed as argument to the "learn" instruction) is also enabled for the
8745          * current learner table.
8746          */
8747         for (i = 0; i < action->n_instructions; i++) {
8748                 struct instruction *instr = &action->instructions[i];
8749                 uint32_t found = 0, j;
8750
8751                 if (instr->type != INSTR_LEARNER_LEARN)
8752                         continue;
8753
8754                 for (j = 0; j < n_actions; j++) {
8755                         struct action *a;
8756
8757                         a = action_find(p, action_names[j]);
8758                         if (!a)
8759                                 return -EINVAL;
8760
8761                         if (a->id == instr->learn.action_id)
8762                                 found = 1;
8763                 }
8764
8765                 if (!found)
8766                         return -EINVAL;
8767         }
8768
8769         return 0;
8770 }
8771
8772 int
8773 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8774                               const char *name,
8775                               struct rte_swx_pipeline_learner_params *params,
8776                               uint32_t size,
8777                               uint32_t *timeout,
8778                               uint32_t n_timeouts)
8779 {
8780         struct learner *l = NULL;
8781         struct action *default_action;
8782         struct header *header = NULL;
8783         uint32_t action_data_size_max = 0, i;
8784         int status = 0;
8785
8786         CHECK(p, EINVAL);
8787
8788         CHECK_NAME(name, EINVAL);
8789         CHECK(!table_find(p, name), EEXIST);
8790         CHECK(!selector_find(p, name), EEXIST);
8791         CHECK(!learner_find(p, name), EEXIST);
8792
8793         CHECK(params, EINVAL);
8794
8795         /* Match checks. */
8796         status = learner_match_fields_check(p, params, &header);
8797         if (status)
8798                 return status;
8799
8800         /* Action checks. */
8801         CHECK(params->n_actions, EINVAL);
8802         CHECK(params->action_names, EINVAL);
8803         for (i = 0; i < params->n_actions; i++) {
8804                 const char *action_name = params->action_names[i];
8805                 struct action *a;
8806                 uint32_t action_data_size;
8807                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8808
8809                 CHECK_NAME(action_name, EINVAL);
8810
8811                 a = action_find(p, action_name);
8812                 CHECK(a, EINVAL);
8813
8814                 status = learner_action_learning_check(p,
8815                                                        a,
8816                                                        params->action_names,
8817                                                        params->n_actions);
8818                 if (status)
8819                         return status;
8820
8821                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8822                 if (action_data_size > action_data_size_max)
8823                         action_data_size_max = action_data_size;
8824
8825                 if (params->action_is_for_table_entries)
8826                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8827                 if (params->action_is_for_default_entry)
8828                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8829                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8830         }
8831
8832         CHECK_NAME(params->default_action_name, EINVAL);
8833         for (i = 0; i < p->n_actions; i++)
8834                 if (!strcmp(params->action_names[i],
8835                             params->default_action_name))
8836                         break;
8837         CHECK(i < params->n_actions, EINVAL);
8838         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8839               EINVAL);
8840
8841         default_action = action_find(p, params->default_action_name);
8842         CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
8843               EINVAL);
8844
8845         /* Any other checks. */
8846         CHECK(size, EINVAL);
8847         CHECK(timeout, EINVAL);
8848         CHECK(n_timeouts && (n_timeouts <= RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX), EINVAL);
8849
8850         /* Memory allocation. */
8851         l = calloc(1, sizeof(struct learner));
8852         if (!l) {
8853                 status = -ENOMEM;
8854                 goto error;
8855         }
8856
8857         l->fields = calloc(params->n_fields, sizeof(struct field *));
8858         if (!l->fields) {
8859                 status = -ENOMEM;
8860                 goto error;
8861         }
8862
8863         l->actions = calloc(params->n_actions, sizeof(struct action *));
8864         if (!l->actions) {
8865                 status = -ENOMEM;
8866                 goto error;
8867         }
8868
8869         if (action_data_size_max) {
8870                 l->default_action_data = calloc(1, action_data_size_max);
8871                 if (!l->default_action_data) {
8872                         status = -ENOMEM;
8873                         goto error;
8874                 }
8875         }
8876
8877         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8878         if (!l->action_is_for_table_entries) {
8879                 status = -ENOMEM;
8880                 goto error;
8881         }
8882
8883         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8884         if (!l->action_is_for_default_entry) {
8885                 status = -ENOMEM;
8886                 goto error;
8887         }
8888
8889         /* Node initialization. */
8890         strcpy(l->name, name);
8891
8892         for (i = 0; i < params->n_fields; i++) {
8893                 const char *field_name = params->field_names[i];
8894
8895                 l->fields[i] = header ?
8896                         header_field_parse(p, field_name, NULL) :
8897                         metadata_field_parse(p, field_name);
8898         }
8899
8900         l->n_fields = params->n_fields;
8901
8902         l->header = header;
8903
8904         for (i = 0; i < params->n_actions; i++) {
8905                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8906
8907                 if (params->action_is_for_table_entries)
8908                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8909                 if (params->action_is_for_default_entry)
8910                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8911
8912                 l->actions[i] = action_find(p, params->action_names[i]);
8913                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8914                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8915         }
8916
8917         l->default_action = default_action;
8918
8919         if (default_action->st) {
8920                 status = action_args_parse(default_action,
8921                                            params->default_action_args,
8922                                            l->default_action_data);
8923                 if (status)
8924                         goto error;
8925         }
8926
8927         l->n_actions = params->n_actions;
8928
8929         l->default_action_is_const = params->default_action_is_const;
8930
8931         l->action_data_size_max = action_data_size_max;
8932
8933         l->size = size;
8934
8935         for (i = 0; i < n_timeouts; i++)
8936                 l->timeout[i] = timeout[i];
8937
8938         l->n_timeouts = n_timeouts;
8939
8940         l->id = p->n_learners;
8941
8942         /* Node add to tailq. */
8943         TAILQ_INSERT_TAIL(&p->learners, l, node);
8944         p->n_learners++;
8945
8946         return 0;
8947
8948 error:
8949         if (!l)
8950                 return status;
8951
8952         free(l->action_is_for_default_entry);
8953         free(l->action_is_for_table_entries);
8954         free(l->default_action_data);
8955         free(l->actions);
8956         free(l->fields);
8957         free(l);
8958
8959         return status;
8960 }
8961
8962 static void
8963 learner_params_free(struct rte_swx_table_learner_params *params)
8964 {
8965         if (!params)
8966                 return;
8967
8968         free(params->key_mask0);
8969
8970         free(params->key_timeout);
8971
8972         free(params);
8973 }
8974
8975 static struct rte_swx_table_learner_params *
8976 learner_params_get(struct learner *l)
8977 {
8978         struct rte_swx_table_learner_params *params = NULL;
8979         struct field *first, *last;
8980         uint32_t i;
8981
8982         /* Memory allocation. */
8983         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8984         if (!params)
8985                 goto error;
8986
8987         /* Find first (smallest offset) and last (biggest offset) match fields. */
8988         first = l->fields[0];
8989         last = l->fields[0];
8990
8991         for (i = 0; i < l->n_fields; i++) {
8992                 struct field *f = l->fields[i];
8993
8994                 if (f->offset < first->offset)
8995                         first = f;
8996
8997                 if (f->offset > last->offset)
8998                         last = f;
8999         }
9000
9001         /* Key offset and size. */
9002         params->key_offset = first->offset / 8;
9003         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9004
9005         /* Memory allocation. */
9006         params->key_mask0 = calloc(1, params->key_size);
9007         if (!params->key_mask0)
9008                 goto error;
9009
9010         /* Key mask. */
9011         for (i = 0; i < l->n_fields; i++) {
9012                 struct field *f = l->fields[i];
9013                 uint32_t start = (f->offset - first->offset) / 8;
9014                 size_t size = f->n_bits / 8;
9015
9016                 memset(&params->key_mask0[start], 0xFF, size);
9017         }
9018
9019         /* Action data size. */
9020         params->action_data_size = l->action_data_size_max;
9021
9022         /* Maximum number of keys. */
9023         params->n_keys_max = l->size;
9024
9025         /* Memory allocation. */
9026         params->key_timeout = calloc(l->n_timeouts, sizeof(uint32_t));
9027         if (!params->key_timeout)
9028                 goto error;
9029
9030         /* Timeout. */
9031         for (i = 0; i < l->n_timeouts; i++)
9032                 params->key_timeout[i] = l->timeout[i];
9033
9034         params->n_key_timeouts = l->n_timeouts;
9035
9036         return params;
9037
9038 error:
9039         learner_params_free(params);
9040         return NULL;
9041 }
9042
9043 static void
9044 learner_build_free(struct rte_swx_pipeline *p)
9045 {
9046         uint32_t i;
9047
9048         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9049                 struct thread *t = &p->threads[i];
9050                 uint32_t j;
9051
9052                 if (!t->learners)
9053                         continue;
9054
9055                 for (j = 0; j < p->n_learners; j++) {
9056                         struct learner_runtime *r = &t->learners[j];
9057
9058                         free(r->mailbox);
9059                 }
9060
9061                 free(t->learners);
9062                 t->learners = NULL;
9063         }
9064
9065         if (p->learner_stats) {
9066                 for (i = 0; i < p->n_learners; i++)
9067                         free(p->learner_stats[i].n_pkts_action);
9068
9069                 free(p->learner_stats);
9070         }
9071 }
9072
9073 static int
9074 learner_build(struct rte_swx_pipeline *p)
9075 {
9076         uint32_t i;
9077         int status = 0;
9078
9079         /* Per pipeline: learner statistics. */
9080         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9081         CHECK(p->learner_stats, ENOMEM);
9082
9083         for (i = 0; i < p->n_learners; i++) {
9084                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9085                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9086         }
9087
9088         /* Per thread: learner run-time. */
9089         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9090                 struct thread *t = &p->threads[i];
9091                 struct learner *l;
9092
9093                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9094                 if (!t->learners) {
9095                         status = -ENOMEM;
9096                         goto error;
9097                 }
9098
9099                 TAILQ_FOREACH(l, &p->learners, node) {
9100                         struct learner_runtime *r = &t->learners[l->id];
9101                         uint64_t size;
9102
9103                         /* r->mailbox. */
9104                         size = rte_swx_table_learner_mailbox_size_get();
9105                         if (size) {
9106                                 r->mailbox = calloc(1, size);
9107                                 if (!r->mailbox) {
9108                                         status = -ENOMEM;
9109                                         goto error;
9110                                 }
9111                         }
9112
9113                         /* r->key. */
9114                         r->key = l->header ?
9115                                 &t->structs[l->header->struct_id] :
9116                                 &t->structs[p->metadata_struct_id];
9117                 }
9118         }
9119
9120         return 0;
9121
9122 error:
9123         learner_build_free(p);
9124         return status;
9125 }
9126
9127 static void
9128 learner_free(struct rte_swx_pipeline *p)
9129 {
9130         learner_build_free(p);
9131
9132         /* Learner tables. */
9133         for ( ; ; ) {
9134                 struct learner *l;
9135
9136                 l = TAILQ_FIRST(&p->learners);
9137                 if (!l)
9138                         break;
9139
9140                 TAILQ_REMOVE(&p->learners, l, node);
9141                 free(l->fields);
9142                 free(l->actions);
9143                 free(l->default_action_data);
9144                 free(l);
9145         }
9146 }
9147
9148 /*
9149  * Table state.
9150  */
9151 static int
9152 table_state_build(struct rte_swx_pipeline *p)
9153 {
9154         struct table *table;
9155         struct selector *s;
9156         struct learner *l;
9157
9158         p->table_state = calloc(p->n_tables + p->n_selectors + p->n_learners,
9159                                 sizeof(struct rte_swx_table_state));
9160         CHECK(p->table_state, ENOMEM);
9161
9162         TAILQ_FOREACH(table, &p->tables, node) {
9163                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9164
9165                 if (table->type) {
9166                         struct rte_swx_table_params *params;
9167
9168                         /* ts->obj. */
9169                         params = table_params_get(table);
9170                         CHECK(params, ENOMEM);
9171
9172                         ts->obj = table->type->ops.create(params,
9173                                 NULL,
9174                                 table->args,
9175                                 p->numa_node);
9176
9177                         table_params_free(params);
9178                         CHECK(ts->obj, ENODEV);
9179                 }
9180
9181                 /* ts->default_action_data. */
9182                 if (table->action_data_size_max) {
9183                         ts->default_action_data =
9184                                 malloc(table->action_data_size_max);
9185                         CHECK(ts->default_action_data, ENOMEM);
9186
9187                         memcpy(ts->default_action_data,
9188                                table->default_action_data,
9189                                table->action_data_size_max);
9190                 }
9191
9192                 /* ts->default_action_id. */
9193                 ts->default_action_id = table->default_action->id;
9194         }
9195
9196         TAILQ_FOREACH(s, &p->selectors, node) {
9197                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9198                 struct rte_swx_table_selector_params *params;
9199
9200                 /* ts->obj. */
9201                 params = selector_table_params_get(s);
9202                 CHECK(params, ENOMEM);
9203
9204                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9205
9206                 selector_params_free(params);
9207                 CHECK(ts->obj, ENODEV);
9208         }
9209
9210         TAILQ_FOREACH(l, &p->learners, node) {
9211                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9212                         p->n_selectors + l->id];
9213                 struct rte_swx_table_learner_params *params;
9214
9215                 /* ts->obj. */
9216                 params = learner_params_get(l);
9217                 CHECK(params, ENOMEM);
9218
9219                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9220                 learner_params_free(params);
9221                 CHECK(ts->obj, ENODEV);
9222
9223                 /* ts->default_action_data. */
9224                 if (l->action_data_size_max) {
9225                         ts->default_action_data = malloc(l->action_data_size_max);
9226                         CHECK(ts->default_action_data, ENOMEM);
9227
9228                         memcpy(ts->default_action_data,
9229                                l->default_action_data,
9230                                l->action_data_size_max);
9231                 }
9232
9233                 /* ts->default_action_id. */
9234                 ts->default_action_id = l->default_action->id;
9235         }
9236
9237         return 0;
9238 }
9239
9240 static void
9241 table_state_build_free(struct rte_swx_pipeline *p)
9242 {
9243         uint32_t i;
9244
9245         if (!p->table_state)
9246                 return;
9247
9248         for (i = 0; i < p->n_tables; i++) {
9249                 struct rte_swx_table_state *ts = &p->table_state[i];
9250                 struct table *table = table_find_by_id(p, i);
9251
9252                 /* ts->obj. */
9253                 if (table->type && ts->obj)
9254                         table->type->ops.free(ts->obj);
9255
9256                 /* ts->default_action_data. */
9257                 free(ts->default_action_data);
9258         }
9259
9260         for (i = 0; i < p->n_selectors; i++) {
9261                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9262
9263                 /* ts->obj. */
9264                 if (ts->obj)
9265                         rte_swx_table_selector_free(ts->obj);
9266         }
9267
9268         for (i = 0; i < p->n_learners; i++) {
9269                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9270
9271                 /* ts->obj. */
9272                 if (ts->obj)
9273                         rte_swx_table_learner_free(ts->obj);
9274
9275                 /* ts->default_action_data. */
9276                 free(ts->default_action_data);
9277         }
9278
9279         free(p->table_state);
9280         p->table_state = NULL;
9281 }
9282
9283 static void
9284 table_state_free(struct rte_swx_pipeline *p)
9285 {
9286         table_state_build_free(p);
9287 }
9288
9289 /*
9290  * Register array.
9291  */
9292 static struct regarray *
9293 regarray_find(struct rte_swx_pipeline *p, const char *name)
9294 {
9295         struct regarray *elem;
9296
9297         TAILQ_FOREACH(elem, &p->regarrays, node)
9298                 if (!strcmp(elem->name, name))
9299                         return elem;
9300
9301         return NULL;
9302 }
9303
9304 static struct regarray *
9305 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9306 {
9307         struct regarray *elem = NULL;
9308
9309         TAILQ_FOREACH(elem, &p->regarrays, node)
9310                 if (elem->id == id)
9311                         return elem;
9312
9313         return NULL;
9314 }
9315
9316 int
9317 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9318                               const char *name,
9319                               uint32_t size,
9320                               uint64_t init_val)
9321 {
9322         struct regarray *r;
9323
9324         CHECK(p, EINVAL);
9325
9326         CHECK_NAME(name, EINVAL);
9327         CHECK(!regarray_find(p, name), EEXIST);
9328
9329         CHECK(size, EINVAL);
9330         size = rte_align32pow2(size);
9331
9332         /* Memory allocation. */
9333         r = calloc(1, sizeof(struct regarray));
9334         CHECK(r, ENOMEM);
9335
9336         /* Node initialization. */
9337         strcpy(r->name, name);
9338         r->init_val = init_val;
9339         r->size = size;
9340         r->id = p->n_regarrays;
9341
9342         /* Node add to tailq. */
9343         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9344         p->n_regarrays++;
9345
9346         return 0;
9347 }
9348
9349 static int
9350 regarray_build(struct rte_swx_pipeline *p)
9351 {
9352         struct regarray *regarray;
9353
9354         if (!p->n_regarrays)
9355                 return 0;
9356
9357         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9358         CHECK(p->regarray_runtime, ENOMEM);
9359
9360         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9361                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9362                 uint32_t i;
9363
9364                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9365                                          RTE_CACHE_LINE_SIZE,
9366                                          p->numa_node);
9367                 CHECK(r->regarray, ENOMEM);
9368
9369                 if (regarray->init_val)
9370                         for (i = 0; i < regarray->size; i++)
9371                                 r->regarray[i] = regarray->init_val;
9372
9373                 r->size_mask = regarray->size - 1;
9374         }
9375
9376         return 0;
9377 }
9378
9379 static void
9380 regarray_build_free(struct rte_swx_pipeline *p)
9381 {
9382         uint32_t i;
9383
9384         if (!p->regarray_runtime)
9385                 return;
9386
9387         for (i = 0; i < p->n_regarrays; i++) {
9388                 struct regarray *regarray = regarray_find_by_id(p, i);
9389                 struct regarray_runtime *r = &p->regarray_runtime[i];
9390
9391                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9392         }
9393
9394         free(p->regarray_runtime);
9395         p->regarray_runtime = NULL;
9396 }
9397
9398 static void
9399 regarray_free(struct rte_swx_pipeline *p)
9400 {
9401         regarray_build_free(p);
9402
9403         for ( ; ; ) {
9404                 struct regarray *elem;
9405
9406                 elem = TAILQ_FIRST(&p->regarrays);
9407                 if (!elem)
9408                         break;
9409
9410                 TAILQ_REMOVE(&p->regarrays, elem, node);
9411                 free(elem);
9412         }
9413 }
9414
9415 /*
9416  * Meter array.
9417  */
9418 static struct meter_profile *
9419 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9420 {
9421         struct meter_profile *elem;
9422
9423         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9424                 if (!strcmp(elem->name, name))
9425                         return elem;
9426
9427         return NULL;
9428 }
9429
9430 static struct metarray *
9431 metarray_find(struct rte_swx_pipeline *p, const char *name)
9432 {
9433         struct metarray *elem;
9434
9435         TAILQ_FOREACH(elem, &p->metarrays, node)
9436                 if (!strcmp(elem->name, name))
9437                         return elem;
9438
9439         return NULL;
9440 }
9441
9442 static struct metarray *
9443 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9444 {
9445         struct metarray *elem = NULL;
9446
9447         TAILQ_FOREACH(elem, &p->metarrays, node)
9448                 if (elem->id == id)
9449                         return elem;
9450
9451         return NULL;
9452 }
9453
9454 int
9455 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9456                                  const char *name,
9457                                  uint32_t size)
9458 {
9459         struct metarray *m;
9460
9461         CHECK(p, EINVAL);
9462
9463         CHECK_NAME(name, EINVAL);
9464         CHECK(!metarray_find(p, name), EEXIST);
9465
9466         CHECK(size, EINVAL);
9467         size = rte_align32pow2(size);
9468
9469         /* Memory allocation. */
9470         m = calloc(1, sizeof(struct metarray));
9471         CHECK(m, ENOMEM);
9472
9473         /* Node initialization. */
9474         strcpy(m->name, name);
9475         m->size = size;
9476         m->id = p->n_metarrays;
9477
9478         /* Node add to tailq. */
9479         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9480         p->n_metarrays++;
9481
9482         return 0;
9483 }
9484
9485 struct meter_profile meter_profile_default = {
9486         .node = {0},
9487         .name = "",
9488         .params = {0},
9489
9490         .profile = {
9491                 .cbs = 10000,
9492                 .pbs = 10000,
9493                 .cir_period = 1,
9494                 .cir_bytes_per_period = 1,
9495                 .pir_period = 1,
9496                 .pir_bytes_per_period = 1,
9497         },
9498
9499         .n_users = 0,
9500 };
9501
9502 static void
9503 meter_init(struct meter *m)
9504 {
9505         memset(m, 0, sizeof(struct meter));
9506         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9507         m->profile = &meter_profile_default;
9508         m->color_mask = RTE_COLOR_GREEN;
9509
9510         meter_profile_default.n_users++;
9511 }
9512
9513 static int
9514 metarray_build(struct rte_swx_pipeline *p)
9515 {
9516         struct metarray *m;
9517
9518         if (!p->n_metarrays)
9519                 return 0;
9520
9521         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9522         CHECK(p->metarray_runtime, ENOMEM);
9523
9524         TAILQ_FOREACH(m, &p->metarrays, node) {
9525                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9526                 uint32_t i;
9527
9528                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9529                                          RTE_CACHE_LINE_SIZE,
9530                                          p->numa_node);
9531                 CHECK(r->metarray, ENOMEM);
9532
9533                 for (i = 0; i < m->size; i++)
9534                         meter_init(&r->metarray[i]);
9535
9536                 r->size_mask = m->size - 1;
9537         }
9538
9539         return 0;
9540 }
9541
9542 static void
9543 metarray_build_free(struct rte_swx_pipeline *p)
9544 {
9545         uint32_t i;
9546
9547         if (!p->metarray_runtime)
9548                 return;
9549
9550         for (i = 0; i < p->n_metarrays; i++) {
9551                 struct metarray *m = metarray_find_by_id(p, i);
9552                 struct metarray_runtime *r = &p->metarray_runtime[i];
9553
9554                 env_free(r->metarray, m->size * sizeof(struct meter));
9555         }
9556
9557         free(p->metarray_runtime);
9558         p->metarray_runtime = NULL;
9559 }
9560
9561 static void
9562 metarray_free(struct rte_swx_pipeline *p)
9563 {
9564         metarray_build_free(p);
9565
9566         /* Meter arrays. */
9567         for ( ; ; ) {
9568                 struct metarray *elem;
9569
9570                 elem = TAILQ_FIRST(&p->metarrays);
9571                 if (!elem)
9572                         break;
9573
9574                 TAILQ_REMOVE(&p->metarrays, elem, node);
9575                 free(elem);
9576         }
9577
9578         /* Meter profiles. */
9579         for ( ; ; ) {
9580                 struct meter_profile *elem;
9581
9582                 elem = TAILQ_FIRST(&p->meter_profiles);
9583                 if (!elem)
9584                         break;
9585
9586                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9587                 free(elem);
9588         }
9589 }
9590
9591 /*
9592  * Pipeline.
9593  */
9594 void
9595 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9596 {
9597         void *lib;
9598
9599         if (!p)
9600                 return;
9601
9602         lib = p->lib;
9603
9604         free(p->instruction_data);
9605         free(p->instructions);
9606
9607         metarray_free(p);
9608         regarray_free(p);
9609         table_state_free(p);
9610         learner_free(p);
9611         selector_free(p);
9612         table_free(p);
9613         action_free(p);
9614         instruction_table_free(p);
9615         metadata_free(p);
9616         header_free(p);
9617         hash_func_free(p);
9618         extern_func_free(p);
9619         extern_obj_free(p);
9620         mirroring_free(p);
9621         port_out_free(p);
9622         port_in_free(p);
9623         struct_free(p);
9624
9625         free(p);
9626
9627         if (lib)
9628                 dlclose(lib);
9629 }
9630
9631 static int
9632 port_in_types_register(struct rte_swx_pipeline *p)
9633 {
9634         int status;
9635
9636         status = rte_swx_pipeline_port_in_type_register(p,
9637                 "ethdev",
9638                 &rte_swx_port_ethdev_reader_ops);
9639         if (status)
9640                 return status;
9641
9642         status = rte_swx_pipeline_port_in_type_register(p,
9643                 "ring",
9644                 &rte_swx_port_ring_reader_ops);
9645         if (status)
9646                 return status;
9647
9648 #ifdef RTE_PORT_PCAP
9649         status = rte_swx_pipeline_port_in_type_register(p,
9650                 "source",
9651                 &rte_swx_port_source_ops);
9652         if (status)
9653                 return status;
9654 #endif
9655
9656         status = rte_swx_pipeline_port_in_type_register(p,
9657                 "fd",
9658                 &rte_swx_port_fd_reader_ops);
9659         if (status)
9660                 return status;
9661
9662         return 0;
9663 }
9664
9665 static int
9666 port_out_types_register(struct rte_swx_pipeline *p)
9667 {
9668         int status;
9669
9670         status = rte_swx_pipeline_port_out_type_register(p,
9671                 "ethdev",
9672                 &rte_swx_port_ethdev_writer_ops);
9673         if (status)
9674                 return status;
9675
9676         status = rte_swx_pipeline_port_out_type_register(p,
9677                 "ring",
9678                 &rte_swx_port_ring_writer_ops);
9679         if (status)
9680                 return status;
9681
9682         status = rte_swx_pipeline_port_out_type_register(p,
9683                 "sink",
9684                 &rte_swx_port_sink_ops);
9685         if (status)
9686                 return status;
9687
9688         status = rte_swx_pipeline_port_out_type_register(p,
9689                 "fd",
9690                 &rte_swx_port_fd_writer_ops);
9691         if (status)
9692                 return status;
9693
9694         return 0;
9695 }
9696
9697 static int
9698 table_types_register(struct rte_swx_pipeline *p)
9699 {
9700         int status;
9701
9702         status = rte_swx_pipeline_table_type_register(p,
9703                 "exact",
9704                 RTE_SWX_TABLE_MATCH_EXACT,
9705                 &rte_swx_table_exact_match_ops);
9706         if (status)
9707                 return status;
9708
9709         status = rte_swx_pipeline_table_type_register(p,
9710                 "wildcard",
9711                 RTE_SWX_TABLE_MATCH_WILDCARD,
9712                 &rte_swx_table_wildcard_match_ops);
9713         if (status)
9714                 return status;
9715
9716         return 0;
9717 }
9718
9719 static int
9720 hash_funcs_register(struct rte_swx_pipeline *p)
9721 {
9722         int status;
9723
9724         status = rte_swx_pipeline_hash_func_register(p, "jhash", rte_jhash);
9725         if (status)
9726                 return status;
9727
9728         status = rte_swx_pipeline_hash_func_register(p, "crc32", rte_hash_crc);
9729         if (status)
9730                 return status;
9731
9732         return 0;
9733 }
9734
9735 int
9736 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9737 {
9738         struct rte_swx_pipeline *pipeline = NULL;
9739         int status = 0;
9740
9741         /* Check input parameters. */
9742         CHECK(p, EINVAL);
9743
9744         /* Memory allocation. */
9745         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9746         if (!pipeline) {
9747                 status = -ENOMEM;
9748                 goto error;
9749         }
9750
9751         /* Initialization. */
9752         TAILQ_INIT(&pipeline->struct_types);
9753         TAILQ_INIT(&pipeline->port_in_types);
9754         TAILQ_INIT(&pipeline->ports_in);
9755         TAILQ_INIT(&pipeline->port_out_types);
9756         TAILQ_INIT(&pipeline->ports_out);
9757         TAILQ_INIT(&pipeline->extern_types);
9758         TAILQ_INIT(&pipeline->extern_objs);
9759         TAILQ_INIT(&pipeline->extern_funcs);
9760         TAILQ_INIT(&pipeline->hash_funcs);
9761         TAILQ_INIT(&pipeline->headers);
9762         TAILQ_INIT(&pipeline->actions);
9763         TAILQ_INIT(&pipeline->table_types);
9764         TAILQ_INIT(&pipeline->tables);
9765         TAILQ_INIT(&pipeline->selectors);
9766         TAILQ_INIT(&pipeline->learners);
9767         TAILQ_INIT(&pipeline->regarrays);
9768         TAILQ_INIT(&pipeline->meter_profiles);
9769         TAILQ_INIT(&pipeline->metarrays);
9770
9771         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9772         pipeline->n_mirroring_slots = RTE_SWX_PACKET_MIRRORING_SLOTS_DEFAULT;
9773         pipeline->n_mirroring_sessions = RTE_SWX_PACKET_MIRRORING_SESSIONS_DEFAULT;
9774         pipeline->numa_node = numa_node;
9775
9776         status = port_in_types_register(pipeline);
9777         if (status)
9778                 goto error;
9779
9780         status = port_out_types_register(pipeline);
9781         if (status)
9782                 goto error;
9783
9784         status = table_types_register(pipeline);
9785         if (status)
9786                 goto error;
9787
9788         status = hash_funcs_register(pipeline);
9789         if (status)
9790                 goto error;
9791
9792         *p = pipeline;
9793         return 0;
9794
9795 error:
9796         rte_swx_pipeline_free(pipeline);
9797         return status;
9798 }
9799
9800 int
9801 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9802                                      const char **instructions,
9803                                      uint32_t n_instructions)
9804 {
9805         int err;
9806         uint32_t i;
9807
9808         err = instruction_config(p, NULL, instructions, n_instructions);
9809         if (err)
9810                 return err;
9811
9812         /* Thread instruction pointer reset. */
9813         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9814                 struct thread *t = &p->threads[i];
9815
9816                 thread_ip_reset(p, t);
9817         }
9818
9819         return 0;
9820 }
9821
9822 static int
9823 pipeline_compile(struct rte_swx_pipeline *p);
9824
9825 int
9826 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9827 {
9828         struct rte_swx_port_sink_params drop_port_params = {
9829                 .file_name = NULL,
9830         };
9831         int status;
9832
9833         CHECK(p, EINVAL);
9834         CHECK(p->build_done == 0, EEXIST);
9835
9836         status = port_in_build(p);
9837         if (status)
9838                 goto error;
9839
9840         /* Drop port. */
9841         status = rte_swx_pipeline_port_out_config(p,
9842                                                   p->n_ports_out,
9843                                                   "sink",
9844                                                   &drop_port_params);
9845         if (status)
9846                 goto error;
9847
9848         status = port_out_build(p);
9849         if (status)
9850                 goto error;
9851
9852         status = mirroring_build(p);
9853         if (status)
9854                 goto error;
9855
9856         status = struct_build(p);
9857         if (status)
9858                 goto error;
9859
9860         status = extern_obj_build(p);
9861         if (status)
9862                 goto error;
9863
9864         status = extern_func_build(p);
9865         if (status)
9866                 goto error;
9867
9868         status = hash_func_build(p);
9869         if (status)
9870                 goto error;
9871
9872         status = header_build(p);
9873         if (status)
9874                 goto error;
9875
9876         status = metadata_build(p);
9877         if (status)
9878                 goto error;
9879
9880         status = instruction_table_build(p);
9881         if (status)
9882                 goto error;
9883
9884         status = action_build(p);
9885         if (status)
9886                 goto error;
9887
9888         status = table_build(p);
9889         if (status)
9890                 goto error;
9891
9892         status = selector_build(p);
9893         if (status)
9894                 goto error;
9895
9896         status = learner_build(p);
9897         if (status)
9898                 goto error;
9899
9900         status = table_state_build(p);
9901         if (status)
9902                 goto error;
9903
9904         status = regarray_build(p);
9905         if (status)
9906                 goto error;
9907
9908         status = metarray_build(p);
9909         if (status)
9910                 goto error;
9911
9912         p->build_done = 1;
9913
9914         pipeline_compile(p);
9915
9916         return 0;
9917
9918 error:
9919         metarray_build_free(p);
9920         regarray_build_free(p);
9921         table_state_build_free(p);
9922         learner_build_free(p);
9923         selector_build_free(p);
9924         table_build_free(p);
9925         action_build_free(p);
9926         instruction_table_build_free(p);
9927         metadata_build_free(p);
9928         header_build_free(p);
9929         hash_func_build_free(p);
9930         extern_func_build_free(p);
9931         extern_obj_build_free(p);
9932         mirroring_build_free(p);
9933         port_out_build_free(p);
9934         port_in_build_free(p);
9935         struct_build_free(p);
9936
9937         return status;
9938 }
9939
9940 void
9941 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9942 {
9943         uint32_t i;
9944
9945         for (i = 0; i < n_instructions; i++)
9946                 instr_exec(p);
9947 }
9948
9949 void
9950 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9951 {
9952         uint32_t i;
9953
9954         for (i = 0; i < p->n_ports_out; i++) {
9955                 struct port_out_runtime *port = &p->out[i];
9956
9957                 if (port->flush)
9958                         port->flush(port->obj);
9959         }
9960 }
9961
9962 /*
9963  * Control.
9964  */
9965 int
9966 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9967                               struct rte_swx_ctl_pipeline_info *pipeline)
9968 {
9969         struct action *action;
9970         struct table *table;
9971         uint32_t n_actions = 0, n_tables = 0;
9972
9973         if (!p || !pipeline)
9974                 return -EINVAL;
9975
9976         TAILQ_FOREACH(action, &p->actions, node)
9977                 n_actions++;
9978
9979         TAILQ_FOREACH(table, &p->tables, node)
9980                 n_tables++;
9981
9982         pipeline->n_ports_in = p->n_ports_in;
9983         pipeline->n_ports_out = p->n_ports_out;
9984         pipeline->n_mirroring_slots = p->n_mirroring_slots;
9985         pipeline->n_mirroring_sessions = p->n_mirroring_sessions;
9986         pipeline->n_actions = n_actions;
9987         pipeline->n_tables = n_tables;
9988         pipeline->n_selectors = p->n_selectors;
9989         pipeline->n_learners = p->n_learners;
9990         pipeline->n_regarrays = p->n_regarrays;
9991         pipeline->n_metarrays = p->n_metarrays;
9992
9993         return 0;
9994 }
9995
9996 int
9997 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9998 {
9999         if (!p || !numa_node)
10000                 return -EINVAL;
10001
10002         *numa_node = p->numa_node;
10003         return 0;
10004 }
10005
10006 int
10007 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10008                             uint32_t action_id,
10009                             struct rte_swx_ctl_action_info *action)
10010 {
10011         struct action *a = NULL;
10012
10013         if (!p || (action_id >= p->n_actions) || !action)
10014                 return -EINVAL;
10015
10016         a = action_find_by_id(p, action_id);
10017         if (!a)
10018                 return -EINVAL;
10019
10020         strcpy(action->name, a->name);
10021         action->n_args = a->st ? a->st->n_fields : 0;
10022         return 0;
10023 }
10024
10025 int
10026 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10027                                 uint32_t action_id,
10028                                 uint32_t action_arg_id,
10029                                 struct rte_swx_ctl_action_arg_info *action_arg)
10030 {
10031         struct action *a = NULL;
10032         struct field *arg = NULL;
10033
10034         if (!p || (action_id >= p->n_actions) || !action_arg)
10035                 return -EINVAL;
10036
10037         a = action_find_by_id(p, action_id);
10038         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10039                 return -EINVAL;
10040
10041         arg = &a->st->fields[action_arg_id];
10042         strcpy(action_arg->name, arg->name);
10043         action_arg->n_bits = arg->n_bits;
10044         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10045
10046         return 0;
10047 }
10048
10049 int
10050 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10051                            uint32_t table_id,
10052                            struct rte_swx_ctl_table_info *table)
10053 {
10054         struct table *t = NULL;
10055
10056         if (!p || !table)
10057                 return -EINVAL;
10058
10059         t = table_find_by_id(p, table_id);
10060         if (!t)
10061                 return -EINVAL;
10062
10063         strcpy(table->name, t->name);
10064         strcpy(table->args, t->args);
10065         table->n_match_fields = t->n_fields;
10066         table->n_actions = t->n_actions;
10067         table->default_action_is_const = t->default_action_is_const;
10068         table->size = t->size;
10069         return 0;
10070 }
10071
10072 int
10073 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10074         uint32_t table_id,
10075         uint32_t match_field_id,
10076         struct rte_swx_ctl_table_match_field_info *match_field)
10077 {
10078         struct table *t;
10079         struct match_field *f;
10080
10081         if (!p || (table_id >= p->n_tables) || !match_field)
10082                 return -EINVAL;
10083
10084         t = table_find_by_id(p, table_id);
10085         if (!t || (match_field_id >= t->n_fields))
10086                 return -EINVAL;
10087
10088         f = &t->fields[match_field_id];
10089         match_field->match_type = f->match_type;
10090         match_field->is_header = t->header ? 1 : 0;
10091         match_field->n_bits = f->field->n_bits;
10092         match_field->offset = f->field->offset;
10093
10094         return 0;
10095 }
10096
10097 int
10098 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10099         uint32_t table_id,
10100         uint32_t table_action_id,
10101         struct rte_swx_ctl_table_action_info *table_action)
10102 {
10103         struct table *t;
10104
10105         if (!p || (table_id >= p->n_tables) || !table_action)
10106                 return -EINVAL;
10107
10108         t = table_find_by_id(p, table_id);
10109         if (!t || (table_action_id >= t->n_actions))
10110                 return -EINVAL;
10111
10112         table_action->action_id = t->actions[table_action_id]->id;
10113
10114         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
10115         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
10116
10117         return 0;
10118 }
10119
10120 int
10121 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10122                           uint32_t table_id,
10123                           struct rte_swx_table_ops *table_ops,
10124                           int *is_stub)
10125 {
10126         struct table *t;
10127
10128         if (!p || (table_id >= p->n_tables))
10129                 return -EINVAL;
10130
10131         t = table_find_by_id(p, table_id);
10132         if (!t)
10133                 return -EINVAL;
10134
10135         if (t->type) {
10136                 if (table_ops)
10137                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10138                 *is_stub = 0;
10139         } else {
10140                 *is_stub = 1;
10141         }
10142
10143         return 0;
10144 }
10145
10146 int
10147 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10148                               uint32_t selector_id,
10149                               struct rte_swx_ctl_selector_info *selector)
10150 {
10151         struct selector *s = NULL;
10152
10153         if (!p || !selector)
10154                 return -EINVAL;
10155
10156         s = selector_find_by_id(p, selector_id);
10157         if (!s)
10158                 return -EINVAL;
10159
10160         strcpy(selector->name, s->name);
10161
10162         selector->n_selector_fields = s->n_selector_fields;
10163         selector->n_groups_max = s->n_groups_max;
10164         selector->n_members_per_group_max = s->n_members_per_group_max;
10165
10166         return 0;
10167 }
10168
10169 int
10170 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10171          uint32_t selector_id,
10172          struct rte_swx_ctl_table_match_field_info *field)
10173 {
10174         struct selector *s;
10175
10176         if (!p || (selector_id >= p->n_selectors) || !field)
10177                 return -EINVAL;
10178
10179         s = selector_find_by_id(p, selector_id);
10180         if (!s)
10181                 return -EINVAL;
10182
10183         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10184         field->is_header = 0;
10185         field->n_bits = s->group_id_field->n_bits;
10186         field->offset = s->group_id_field->offset;
10187
10188         return 0;
10189 }
10190
10191 int
10192 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10193          uint32_t selector_id,
10194          uint32_t selector_field_id,
10195          struct rte_swx_ctl_table_match_field_info *field)
10196 {
10197         struct selector *s;
10198         struct field *f;
10199
10200         if (!p || (selector_id >= p->n_selectors) || !field)
10201                 return -EINVAL;
10202
10203         s = selector_find_by_id(p, selector_id);
10204         if (!s || (selector_field_id >= s->n_selector_fields))
10205                 return -EINVAL;
10206
10207         f = s->selector_fields[selector_field_id];
10208         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10209         field->is_header = s->selector_header ? 1 : 0;
10210         field->n_bits = f->n_bits;
10211         field->offset = f->offset;
10212
10213         return 0;
10214 }
10215
10216 int
10217 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10218          uint32_t selector_id,
10219          struct rte_swx_ctl_table_match_field_info *field)
10220 {
10221         struct selector *s;
10222
10223         if (!p || (selector_id >= p->n_selectors) || !field)
10224                 return -EINVAL;
10225
10226         s = selector_find_by_id(p, selector_id);
10227         if (!s)
10228                 return -EINVAL;
10229
10230         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10231         field->is_header = 0;
10232         field->n_bits = s->member_id_field->n_bits;
10233         field->offset = s->member_id_field->offset;
10234
10235         return 0;
10236 }
10237
10238 int
10239 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10240                              uint32_t learner_id,
10241                              struct rte_swx_ctl_learner_info *learner)
10242 {
10243         struct learner *l = NULL;
10244
10245         if (!p || !learner)
10246                 return -EINVAL;
10247
10248         l = learner_find_by_id(p, learner_id);
10249         if (!l)
10250                 return -EINVAL;
10251
10252         strcpy(learner->name, l->name);
10253
10254         learner->n_match_fields = l->n_fields;
10255         learner->n_actions = l->n_actions;
10256         learner->default_action_is_const = l->default_action_is_const;
10257         learner->size = l->size;
10258         learner->n_key_timeouts = l->n_timeouts;
10259
10260         return 0;
10261 }
10262
10263 int
10264 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10265                                          uint32_t learner_id,
10266                                          uint32_t match_field_id,
10267                                          struct rte_swx_ctl_table_match_field_info *match_field)
10268 {
10269         struct learner *l;
10270         struct field *f;
10271
10272         if (!p || (learner_id >= p->n_learners) || !match_field)
10273                 return -EINVAL;
10274
10275         l = learner_find_by_id(p, learner_id);
10276         if (!l || (match_field_id >= l->n_fields))
10277                 return -EINVAL;
10278
10279         f = l->fields[match_field_id];
10280         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10281         match_field->is_header = l->header ? 1 : 0;
10282         match_field->n_bits = f->n_bits;
10283         match_field->offset = f->offset;
10284
10285         return 0;
10286 }
10287
10288 int
10289 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10290                                     uint32_t learner_id,
10291                                     uint32_t learner_action_id,
10292                                     struct rte_swx_ctl_table_action_info *learner_action)
10293 {
10294         struct learner *l;
10295
10296         if (!p || (learner_id >= p->n_learners) || !learner_action)
10297                 return -EINVAL;
10298
10299         l = learner_find_by_id(p, learner_id);
10300         if (!l || (learner_action_id >= l->n_actions))
10301                 return -EINVAL;
10302
10303         learner_action->action_id = l->actions[learner_action_id]->id;
10304
10305         learner_action->action_is_for_table_entries =
10306                 l->action_is_for_table_entries[learner_action_id];
10307
10308         learner_action->action_is_for_default_entry =
10309                 l->action_is_for_default_entry[learner_action_id];
10310
10311         return 0;
10312 }
10313
10314 int
10315 rte_swx_ctl_pipeline_learner_timeout_get(struct rte_swx_pipeline *p,
10316                                          uint32_t learner_id,
10317                                          uint32_t timeout_id,
10318                                          uint32_t *timeout)
10319 {
10320         struct learner *l;
10321
10322         if (!p || (learner_id >= p->n_learners) || !timeout)
10323                 return -EINVAL;
10324
10325         l = learner_find_by_id(p, learner_id);
10326         if (!l || (timeout_id >= l->n_timeouts))
10327                 return -EINVAL;
10328
10329         *timeout = l->timeout[timeout_id];
10330         return 0;
10331 }
10332
10333 int
10334 rte_swx_ctl_pipeline_learner_timeout_set(struct rte_swx_pipeline *p,
10335                                          uint32_t learner_id,
10336                                          uint32_t timeout_id,
10337                                          uint32_t timeout)
10338 {
10339         struct learner *l;
10340         struct rte_swx_table_state *ts;
10341         int status;
10342
10343         if (!p || (learner_id >= p->n_learners) || !timeout)
10344                 return -EINVAL;
10345
10346         l = learner_find_by_id(p, learner_id);
10347         if (!l || (timeout_id >= l->n_timeouts))
10348                 return -EINVAL;
10349
10350         if (!p->build_done)
10351                 return -EINVAL;
10352
10353         ts = &p->table_state[p->n_tables + p->n_selectors + l->id];
10354
10355         status = rte_swx_table_learner_timeout_update(ts->obj, timeout_id, timeout);
10356         if (status)
10357                 return -EINVAL;
10358
10359         l->timeout[timeout_id] = timeout;
10360
10361         return 0;
10362 }
10363
10364 int
10365 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10366                                  struct rte_swx_table_state **table_state)
10367 {
10368         if (!p || !table_state || !p->build_done)
10369                 return -EINVAL;
10370
10371         *table_state = p->table_state;
10372         return 0;
10373 }
10374
10375 int
10376 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10377                                  struct rte_swx_table_state *table_state)
10378 {
10379         if (!p || !table_state || !p->build_done)
10380                 return -EINVAL;
10381
10382         p->table_state = table_state;
10383         return 0;
10384 }
10385
10386 int
10387 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10388                                         uint32_t port_id,
10389                                         struct rte_swx_port_in_stats *stats)
10390 {
10391         struct port_in *port;
10392
10393         if (!p || !stats)
10394                 return -EINVAL;
10395
10396         port = port_in_find(p, port_id);
10397         if (!port)
10398                 return -EINVAL;
10399
10400         port->type->ops.stats_read(port->obj, stats);
10401         return 0;
10402 }
10403
10404 int
10405 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10406                                          uint32_t port_id,
10407                                          struct rte_swx_port_out_stats *stats)
10408 {
10409         struct port_out *port;
10410
10411         if (!p || !stats)
10412                 return -EINVAL;
10413
10414         port = port_out_find(p, port_id);
10415         if (!port)
10416                 return -EINVAL;
10417
10418         port->type->ops.stats_read(port->obj, stats);
10419         return 0;
10420 }
10421
10422 int
10423 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10424                                       const char *table_name,
10425                                       struct rte_swx_table_stats *stats)
10426 {
10427         struct table *table;
10428         struct table_statistics *table_stats;
10429
10430         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10431                 return -EINVAL;
10432
10433         table = table_find(p, table_name);
10434         if (!table)
10435                 return -EINVAL;
10436
10437         table_stats = &p->table_stats[table->id];
10438
10439         memcpy(stats->n_pkts_action,
10440                table_stats->n_pkts_action,
10441                p->n_actions * sizeof(uint64_t));
10442
10443         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10444         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10445
10446         return 0;
10447 }
10448
10449 int
10450 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10451         const char *selector_name,
10452         struct rte_swx_pipeline_selector_stats *stats)
10453 {
10454         struct selector *s;
10455
10456         if (!p || !selector_name || !selector_name[0] || !stats)
10457                 return -EINVAL;
10458
10459         s = selector_find(p, selector_name);
10460         if (!s)
10461                 return -EINVAL;
10462
10463         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10464
10465         return 0;
10466 }
10467
10468 int
10469 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10470                                         const char *learner_name,
10471                                         struct rte_swx_learner_stats *stats)
10472 {
10473         struct learner *l;
10474         struct learner_statistics *learner_stats;
10475
10476         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10477                 return -EINVAL;
10478
10479         l = learner_find(p, learner_name);
10480         if (!l)
10481                 return -EINVAL;
10482
10483         learner_stats = &p->learner_stats[l->id];
10484
10485         memcpy(stats->n_pkts_action,
10486                learner_stats->n_pkts_action,
10487                p->n_actions * sizeof(uint64_t));
10488
10489         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10490         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10491
10492         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10493         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10494
10495         stats->n_pkts_rearm = learner_stats->n_pkts_rearm;
10496         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10497
10498         return 0;
10499 }
10500
10501 int
10502 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10503                               uint32_t regarray_id,
10504                               struct rte_swx_ctl_regarray_info *regarray)
10505 {
10506         struct regarray *r;
10507
10508         if (!p || !regarray)
10509                 return -EINVAL;
10510
10511         r = regarray_find_by_id(p, regarray_id);
10512         if (!r)
10513                 return -EINVAL;
10514
10515         strcpy(regarray->name, r->name);
10516         regarray->size = r->size;
10517         return 0;
10518 }
10519
10520 int
10521 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10522                                    const char *regarray_name,
10523                                    uint32_t regarray_index,
10524                                    uint64_t *value)
10525 {
10526         struct regarray *regarray;
10527         struct regarray_runtime *r;
10528
10529         if (!p || !regarray_name || !value)
10530                 return -EINVAL;
10531
10532         regarray = regarray_find(p, regarray_name);
10533         if (!regarray || (regarray_index >= regarray->size))
10534                 return -EINVAL;
10535
10536         r = &p->regarray_runtime[regarray->id];
10537         *value = r->regarray[regarray_index];
10538         return 0;
10539 }
10540
10541 int
10542 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10543                                    const char *regarray_name,
10544                                    uint32_t regarray_index,
10545                                    uint64_t value)
10546 {
10547         struct regarray *regarray;
10548         struct regarray_runtime *r;
10549
10550         if (!p || !regarray_name)
10551                 return -EINVAL;
10552
10553         regarray = regarray_find(p, regarray_name);
10554         if (!regarray || (regarray_index >= regarray->size))
10555                 return -EINVAL;
10556
10557         r = &p->regarray_runtime[regarray->id];
10558         r->regarray[regarray_index] = value;
10559         return 0;
10560 }
10561
10562 int
10563 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10564                               uint32_t metarray_id,
10565                               struct rte_swx_ctl_metarray_info *metarray)
10566 {
10567         struct metarray *m;
10568
10569         if (!p || !metarray)
10570                 return -EINVAL;
10571
10572         m = metarray_find_by_id(p, metarray_id);
10573         if (!m)
10574                 return -EINVAL;
10575
10576         strcpy(metarray->name, m->name);
10577         metarray->size = m->size;
10578         return 0;
10579 }
10580
10581 int
10582 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10583                               const char *name,
10584                               struct rte_meter_trtcm_params *params)
10585 {
10586         struct meter_profile *mp;
10587         int status;
10588
10589         CHECK(p, EINVAL);
10590         CHECK_NAME(name, EINVAL);
10591         CHECK(params, EINVAL);
10592         CHECK(!meter_profile_find(p, name), EEXIST);
10593
10594         /* Node allocation. */
10595         mp = calloc(1, sizeof(struct meter_profile));
10596         CHECK(mp, ENOMEM);
10597
10598         /* Node initialization. */
10599         strcpy(mp->name, name);
10600         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10601         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10602         if (status) {
10603                 free(mp);
10604                 CHECK(0, EINVAL);
10605         }
10606
10607         /* Node add to tailq. */
10608         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10609
10610         return 0;
10611 }
10612
10613 int
10614 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10615                                  const char *name)
10616 {
10617         struct meter_profile *mp;
10618
10619         CHECK(p, EINVAL);
10620         CHECK_NAME(name, EINVAL);
10621
10622         mp = meter_profile_find(p, name);
10623         CHECK(mp, EINVAL);
10624         CHECK(!mp->n_users, EBUSY);
10625
10626         /* Remove node from tailq. */
10627         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10628         free(mp);
10629
10630         return 0;
10631 }
10632
10633 int
10634 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10635                         const char *metarray_name,
10636                         uint32_t metarray_index)
10637 {
10638         struct meter_profile *mp_old;
10639         struct metarray *metarray;
10640         struct metarray_runtime *metarray_runtime;
10641         struct meter *m;
10642
10643         CHECK(p, EINVAL);
10644         CHECK_NAME(metarray_name, EINVAL);
10645
10646         metarray = metarray_find(p, metarray_name);
10647         CHECK(metarray, EINVAL);
10648         CHECK(metarray_index < metarray->size, EINVAL);
10649
10650         metarray_runtime = &p->metarray_runtime[metarray->id];
10651         m = &metarray_runtime->metarray[metarray_index];
10652         mp_old = m->profile;
10653
10654         meter_init(m);
10655
10656         mp_old->n_users--;
10657
10658         return 0;
10659 }
10660
10661 int
10662 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10663                       const char *metarray_name,
10664                       uint32_t metarray_index,
10665                       const char *profile_name)
10666 {
10667         struct meter_profile *mp, *mp_old;
10668         struct metarray *metarray;
10669         struct metarray_runtime *metarray_runtime;
10670         struct meter *m;
10671
10672         CHECK(p, EINVAL);
10673         CHECK_NAME(metarray_name, EINVAL);
10674
10675         metarray = metarray_find(p, metarray_name);
10676         CHECK(metarray, EINVAL);
10677         CHECK(metarray_index < metarray->size, EINVAL);
10678
10679         mp = meter_profile_find(p, profile_name);
10680         CHECK(mp, EINVAL);
10681
10682         metarray_runtime = &p->metarray_runtime[metarray->id];
10683         m = &metarray_runtime->metarray[metarray_index];
10684         mp_old = m->profile;
10685
10686         memset(m, 0, sizeof(struct meter));
10687         rte_meter_trtcm_config(&m->m, &mp->profile);
10688         m->profile = mp;
10689         m->color_mask = RTE_COLORS;
10690
10691         mp->n_users++;
10692         mp_old->n_users--;
10693
10694         return 0;
10695 }
10696
10697 int
10698 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10699                              const char *metarray_name,
10700                              uint32_t metarray_index,
10701                              struct rte_swx_ctl_meter_stats *stats)
10702 {
10703         struct metarray *metarray;
10704         struct metarray_runtime *metarray_runtime;
10705         struct meter *m;
10706
10707         CHECK(p, EINVAL);
10708         CHECK_NAME(metarray_name, EINVAL);
10709
10710         metarray = metarray_find(p, metarray_name);
10711         CHECK(metarray, EINVAL);
10712         CHECK(metarray_index < metarray->size, EINVAL);
10713
10714         CHECK(stats, EINVAL);
10715
10716         metarray_runtime = &p->metarray_runtime[metarray->id];
10717         m = &metarray_runtime->metarray[metarray_index];
10718
10719         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10720         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10721
10722         return 0;
10723 }
10724
10725 int
10726 rte_swx_ctl_pipeline_mirroring_session_set(struct rte_swx_pipeline *p,
10727                                            uint32_t session_id,
10728                                            struct rte_swx_pipeline_mirroring_session_params *params)
10729 {
10730         struct mirroring_session *s;
10731
10732         CHECK(p, EINVAL);
10733         CHECK(p->build_done, EEXIST);
10734         CHECK(session_id < p->n_mirroring_sessions, EINVAL);
10735         CHECK(params, EINVAL);
10736         CHECK(params->port_id < p->n_ports_out, EINVAL);
10737
10738         s = &p->mirroring_sessions[session_id];
10739         s->port_id = params->port_id;
10740         s->fast_clone = params->fast_clone;
10741         s->truncation_length = params->truncation_length ? params->truncation_length : UINT32_MAX;
10742
10743         return 0;
10744 }
10745
10746 /*
10747  * Pipeline compilation.
10748  */
10749 static const char *
10750 instr_type_to_name(struct instruction *instr)
10751 {
10752         switch (instr->type) {
10753         case INSTR_RX: return "INSTR_RX";
10754
10755         case INSTR_TX: return "INSTR_TX";
10756         case INSTR_TX_I: return "INSTR_TX_I";
10757         case INSTR_DROP: return "INSTR_DROP";
10758         case INSTR_MIRROR: return "INSTR_MIRROR";
10759         case INSTR_RECIRCULATE: return "INSTR_RECIRCULATE";
10760         case INSTR_RECIRCID: return "INSTR_RECIRCID";
10761
10762         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10763         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10764         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10765         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10766         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10767         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10768         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10769         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10770
10771         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10772
10773         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10774
10775         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10776         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10777         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10778         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10779         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10780         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10781         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10782         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10783         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10784
10785         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10786         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10787
10788         case INSTR_MOV: return "INSTR_MOV";
10789         case INSTR_MOV_MH: return "INSTR_MOV_MH";
10790         case INSTR_MOV_HM: return "INSTR_MOV_HM";
10791         case INSTR_MOV_HH: return "INSTR_MOV_HH";
10792         case INSTR_MOV_I: return "INSTR_MOV_I";
10793
10794         case INSTR_DMA_HT: return "INSTR_DMA_HT";
10795         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10796         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10797         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10798         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10799         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10800         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10801         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10802
10803         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10804         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10805         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10806         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10807         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10808         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10809
10810         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10811         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10812         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10813         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10814         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10815         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10816
10817         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10818         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10819         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10820         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10821
10822         case INSTR_ALU_AND: return "INSTR_ALU_AND";
10823         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10824         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10825         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10826         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10827
10828         case INSTR_ALU_OR: return "INSTR_ALU_OR";
10829         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10830         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10831         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10832         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10833
10834         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10835         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10836         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10837         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10838         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10839
10840         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10841         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10842         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10843         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10844         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10845         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10846
10847         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10848         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10849         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10850         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10851         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10852         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10853
10854         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10855         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10856         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10857
10858         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10859         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10860         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10861         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10862         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10863         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10864
10865         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10866         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10867         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10868         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10869         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10870         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10871         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10872         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10873         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10874
10875         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10876         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10877         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10878         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10879         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10880         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10881         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10882         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10883         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10884
10885         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10886         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10887         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10888
10889         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10890         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10891         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10892         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10893         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10894         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10895         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10896         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10897         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10898         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10899         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10900         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10901
10902         case INSTR_TABLE: return "INSTR_TABLE";
10903         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10904         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10905         case INSTR_LEARNER: return "INSTR_LEARNER";
10906         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10907
10908         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10909         case INSTR_LEARNER_REARM: return "INSTR_LEARNER_REARM";
10910         case INSTR_LEARNER_REARM_NEW: return "INSTR_LEARNER_REARM_NEW";
10911         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10912
10913         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10914         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10915         case INSTR_HASH_FUNC: return "INSTR_HASH_FUNC";
10916
10917         case INSTR_JMP: return "INSTR_JMP";
10918         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10919         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10920         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10921         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10922         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10923         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10924         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10925         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10926         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10927         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10928         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10929         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10930         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10931         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10932         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10933         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10934         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10935         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10936         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10937         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10938         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10939         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10940         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10941         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10942         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10943         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10944         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10945         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10946
10947         case INSTR_RETURN: return "INSTR_RETURN";
10948
10949         default: return "INSTR_UNKNOWN";
10950         }
10951 }
10952
10953 typedef void
10954 (*instruction_export_t)(struct instruction *, FILE *);
10955
10956 static void
10957 instr_io_export(struct instruction *instr, FILE *f)
10958 {
10959         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10960
10961         /* n_io, n_io_imm, n_hdrs. */
10962         if (instr->type == INSTR_RX ||
10963             instr->type == INSTR_TX ||
10964             instr->type == INSTR_HDR_EXTRACT_M ||
10965             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10966                 n_io = 1;
10967
10968         if (instr->type == INSTR_TX_I)
10969                 n_io_imm = 1;
10970
10971         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10972                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10973
10974         if (instr->type == INSTR_HDR_EXTRACT_M ||
10975             instr->type == INSTR_HDR_LOOKAHEAD ||
10976             instr->type == INSTR_HDR_EMIT)
10977                 n_hdrs = 1;
10978
10979         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10980                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10981
10982         /* instr. */
10983         fprintf(f,
10984                 "\t{\n"
10985                 "\t\t.type = %s,\n",
10986                 instr_type_to_name(instr));
10987
10988         /* instr.io. */
10989         if (n_io || n_io_imm || n_hdrs)
10990                 fprintf(f,
10991                         "\t\t.io = {\n");
10992
10993         /* instr.io.io. */
10994         if (n_io)
10995                 fprintf(f,
10996                         "\t\t\t.io = {\n"
10997                         "\t\t\t\t.offset = %u,\n"
10998                         "\t\t\t\t.n_bits = %u,\n"
10999                         "\t\t\t},\n",
11000                         instr->io.io.offset,
11001                         instr->io.io.n_bits);
11002
11003         if (n_io_imm)
11004                 fprintf(f,
11005                         "\t\t\t.io = {\n"
11006                         "\t\t\t\t.val = %u,\n"
11007                         "\t\t\t},\n",
11008                         instr->io.io.val);
11009
11010         /* instr.io.hdr. */
11011         if (n_hdrs) {
11012                 fprintf(f,
11013                         "\t\t.hdr = {\n");
11014
11015                 /* instr.io.hdr.header_id. */
11016                 fprintf(f,
11017                         "\t\t\t.header_id = {");
11018
11019                 for (i = 0; i < n_hdrs; i++)
11020                         fprintf(f,
11021                                 "%u, ",
11022                                 instr->io.hdr.header_id[i]);
11023
11024                 fprintf(f,
11025                         "},\n");
11026
11027                 /* instr.io.hdr.struct_id. */
11028                 fprintf(f,
11029                         "\t\t\t.struct_id = {");
11030
11031                 for (i = 0; i < n_hdrs; i++)
11032                         fprintf(f,
11033                                 "%u, ",
11034                                 instr->io.hdr.struct_id[i]);
11035
11036                 fprintf(f,
11037                         "},\n");
11038
11039                 /* instr.io.hdr.n_bytes. */
11040                 fprintf(f,
11041                         "\t\t\t.n_bytes = {");
11042
11043                 for (i = 0; i < n_hdrs; i++)
11044                         fprintf(f,
11045                                 "%u, ",
11046                                 instr->io.hdr.n_bytes[i]);
11047
11048                 fprintf(f,
11049                         "},\n");
11050
11051                 /* instr.io.hdr - closing curly brace. */
11052                 fprintf(f,
11053                         "\t\t\t}\n,");
11054         }
11055
11056         /* instr.io - closing curly brace. */
11057         if (n_io || n_io_imm || n_hdrs)
11058                 fprintf(f,
11059                         "\t\t},\n");
11060
11061         /* instr - closing curly brace. */
11062         fprintf(f,
11063                 "\t},\n");
11064 }
11065
11066 static void
11067 instr_mirror_export(struct instruction *instr, FILE *f)
11068 {
11069         fprintf(f,
11070                 "\t{\n"
11071                 "\t\t.type = %s,\n"
11072                 "\t\t.mirror = {\n"
11073                 "\t\t\t.dst = {\n"
11074                 "\t\t\t\t.struct_id = %u,\n"
11075                 "\t\t\t\t.n_bits = %u,\n"
11076                 "\t\t\t\t.offset = %u,\n"
11077                 "\t\t\t}\n,"
11078                 "\t\t\t.src = {\n"
11079                 "\t\t\t\t.struct_id = %u,\n"
11080                 "\t\t\t\t.n_bits = %u,\n"
11081                 "\t\t\t\t.offset = %u,\n"
11082                 "\t\t\t}\n,"
11083                 "\t\t},\n"
11084                 "\t},\n",
11085                 instr_type_to_name(instr),
11086                 instr->mirror.dst.struct_id,
11087                 instr->mirror.dst.n_bits,
11088                 instr->mirror.dst.offset,
11089                 instr->mirror.src.struct_id,
11090                 instr->mirror.src.n_bits,
11091                 instr->mirror.src.offset);
11092 }
11093
11094 static void
11095 instr_recirculate_export(struct instruction *instr, FILE *f)
11096 {
11097         fprintf(f,
11098                 "\t{\n"
11099                 "\t\t.type = %s,\n"
11100                 "\t},\n",
11101                 instr_type_to_name(instr));
11102 }
11103
11104 static void
11105 instr_recircid_export(struct instruction *instr, FILE *f)
11106 {
11107         fprintf(f,
11108                 "\t{\n"
11109                 "\t\t.type = %s,\n"
11110                 "\t\t.io = {\n"
11111                 "\t\t\t.offset = %u,\n"
11112                 "\t\t\t.n_bits = %u,\n"
11113                 "\t\t},\n"
11114                 "\t},\n",
11115                 instr_type_to_name(instr),
11116                 instr->io.io.offset,
11117                 instr->io.io.n_bits);
11118 }
11119
11120 static void
11121 instr_hdr_validate_export(struct instruction *instr, FILE *f)
11122 {
11123         fprintf(f,
11124                 "\t{\n"
11125                 "\t\t.type = %s,\n"
11126                 "\t\t.valid = {\n"
11127                 "\t\t\t.header_id = %u,\n"
11128                 "\t\t},\n"
11129                 "\t},\n",
11130                 instr_type_to_name(instr),
11131                 instr->valid.header_id);
11132 }
11133
11134 static void
11135 instr_mov_export(struct instruction *instr, FILE *f)
11136 {
11137         if (instr->type != INSTR_MOV_I)
11138                 fprintf(f,
11139                         "\t{\n"
11140                         "\t\t.type = %s,\n"
11141                         "\t\t.mov = {\n"
11142                         "\t\t\t.dst = {\n"
11143                         "\t\t\t\t.struct_id = %u,\n"
11144                         "\t\t\t\t.n_bits = %u,\n"
11145                         "\t\t\t\t.offset = %u,\n"
11146                         "\t\t\t},\n"
11147                         "\t\t\t.src = {\n"
11148                         "\t\t\t\t.struct_id = %u,\n"
11149                         "\t\t\t\t.n_bits = %u,\n"
11150                         "\t\t\t\t.offset = %u,\n"
11151                         "\t\t\t},\n"
11152                         "\t\t},\n"
11153                         "\t},\n",
11154                         instr_type_to_name(instr),
11155                         instr->mov.dst.struct_id,
11156                         instr->mov.dst.n_bits,
11157                         instr->mov.dst.offset,
11158                         instr->mov.src.struct_id,
11159                         instr->mov.src.n_bits,
11160                         instr->mov.src.offset);
11161         else
11162                 fprintf(f,
11163                         "\t{\n"
11164                         "\t\t.type = %s,\n"
11165                         "\t\t.mov = {\n"
11166                         "\t\t\t.dst = {\n"
11167                         "\t\t\t\t.struct_id = %u,\n"
11168                         "\t\t\t\t.n_bits = %u,\n"
11169                         "\t\t\t\t.offset = %u,\n"
11170                         "\t\t\t}\n,"
11171                         "\t\t\t.src_val = %" PRIu64 ",\n"
11172                         "\t\t},\n"
11173                         "\t},\n",
11174                         instr_type_to_name(instr),
11175                         instr->mov.dst.struct_id,
11176                         instr->mov.dst.n_bits,
11177                         instr->mov.dst.offset,
11178                         instr->mov.src_val);
11179 }
11180
11181 static void
11182 instr_dma_ht_export(struct instruction *instr, FILE *f)
11183 {
11184         uint32_t n_dma = 0, i;
11185
11186         /* n_dma. */
11187         n_dma = 1 + (instr->type - INSTR_DMA_HT);
11188
11189         /* instr. */
11190         fprintf(f,
11191                 "\t{\n"
11192                 "\t\t.type = %s,\n",
11193                 instr_type_to_name(instr));
11194
11195         /* instr.dma. */
11196         fprintf(f,
11197                 "\t\t.dma = {\n");
11198
11199         /* instr.dma.dst. */
11200         fprintf(f,
11201                 "\t\t\t.dst = {\n");
11202
11203         /* instr.dma.dst.header_id. */
11204         fprintf(f,
11205                 "\t\t\t\t.header_id = {");
11206
11207         for (i = 0; i < n_dma; i++)
11208                 fprintf(f,
11209                         "%u, ",
11210                         instr->dma.dst.header_id[i]);
11211
11212         fprintf(f,
11213                 "},\n");
11214
11215         /* instr.dma.dst.struct_id. */
11216         fprintf(f,
11217                 "\t\t\t\t.struct_id = {");
11218
11219         for (i = 0; i < n_dma; i++)
11220                 fprintf(f,
11221                         "%u, ",
11222                         instr->dma.dst.struct_id[i]);
11223
11224         fprintf(f,
11225                 "},\n");
11226
11227         /* instr.dma.dst - closing curly brace. */
11228         fprintf(f,
11229                 "\t\t\t},\n");
11230
11231         /* instr.dma.src. */
11232         fprintf(f,
11233                 "\t\t\t.src = {\n");
11234
11235         /* instr.dma.src.offset. */
11236         fprintf(f,
11237                 "\t\t\t\t.offset = {");
11238
11239         for (i = 0; i < n_dma; i++)
11240                 fprintf(f,
11241                         "%u, ",
11242                         instr->dma.src.offset[i]);
11243
11244         fprintf(f,
11245                 "},\n");
11246
11247         /* instr.dma.src - closing curly brace. */
11248         fprintf(f,
11249                 "\t\t\t},\n");
11250
11251         /* instr.dma.n_bytes. */
11252         fprintf(f,
11253                 "\t\t\t.n_bytes = {");
11254
11255         for (i = 0; i < n_dma; i++)
11256                 fprintf(f,
11257                         "%u, ",
11258                         instr->dma.n_bytes[i]);
11259
11260         fprintf(f,
11261                 "},\n");
11262
11263         /* instr.dma - closing curly brace. */
11264         fprintf(f,
11265                 "\t\t},\n");
11266
11267         /* instr - closing curly brace. */
11268         fprintf(f,
11269                 "\t},\n");
11270 }
11271
11272 static void
11273 instr_alu_export(struct instruction *instr, FILE *f)
11274 {
11275         int imm = 0;
11276
11277         if (instr->type == INSTR_ALU_ADD_MI ||
11278             instr->type == INSTR_ALU_ADD_HI ||
11279             instr->type == INSTR_ALU_SUB_MI ||
11280             instr->type == INSTR_ALU_SUB_HI ||
11281             instr->type == INSTR_ALU_SHL_MI ||
11282             instr->type == INSTR_ALU_SHL_HI ||
11283             instr->type == INSTR_ALU_SHR_MI ||
11284             instr->type == INSTR_ALU_SHR_HI ||
11285             instr->type == INSTR_ALU_AND_I ||
11286             instr->type == INSTR_ALU_OR_I ||
11287             instr->type == INSTR_ALU_XOR_I)
11288                 imm = 1;
11289
11290         if (!imm)
11291                 fprintf(f,
11292                         "\t{\n"
11293                         "\t\t.type = %s,\n"
11294                         "\t\t.alu = {\n"
11295                         "\t\t\t.dst = {\n"
11296                         "\t\t\t\t.struct_id = %u,\n"
11297                         "\t\t\t\t.n_bits = %u,\n"
11298                         "\t\t\t\t.offset = %u,\n"
11299                         "\t\t\t},\n"
11300                         "\t\t\t.src = {\n"
11301                         "\t\t\t\t.struct_id = %u,\n"
11302                         "\t\t\t\t.n_bits = %u,\n"
11303                         "\t\t\t\t.offset = %u,\n"
11304                         "\t\t\t},\n"
11305                         "\t\t},\n"
11306                         "\t},\n",
11307                         instr_type_to_name(instr),
11308                         instr->alu.dst.struct_id,
11309                         instr->alu.dst.n_bits,
11310                         instr->alu.dst.offset,
11311                         instr->alu.src.struct_id,
11312                         instr->alu.src.n_bits,
11313                         instr->alu.src.offset);
11314         else
11315                 fprintf(f,
11316                         "\t{\n"
11317                         "\t\t.type = %s,\n"
11318                         "\t\t.alu = {\n"
11319                         "\t\t\t.dst = {\n"
11320                         "\t\t\t\t.struct_id = %u,\n"
11321                         "\t\t\t\t.n_bits = %u,\n"
11322                         "\t\t\t\t.offset = %u,\n"
11323                         "\t\t\t}\n,"
11324                         "\t\t\t.src_val = %" PRIu64 ",\n"
11325                         "\t\t},\n"
11326                         "\t},\n",
11327                         instr_type_to_name(instr),
11328                         instr->alu.dst.struct_id,
11329                         instr->alu.dst.n_bits,
11330                         instr->alu.dst.offset,
11331                         instr->alu.src_val);
11332 }
11333
11334 static void
11335 instr_hash_export(struct instruction *instr, FILE *f)
11336 {
11337         fprintf(f,
11338                 "\t{\n"
11339                 "\t\t.type = %s,\n"
11340                 "\t\t.hash_func = {\n"
11341                 "\t\t\t.hash_func_id = %u,\n"
11342                 "\t\t\t.dst = {\n"
11343                 "\t\t\t\t.offset = %u,\n"
11344                 "\t\t\t\t.n_bits = %u,\n"
11345                 "\t\t\t},\n"
11346                 "\t\t\t.src = {\n"
11347                 "\t\t\t\t.struct_id = %u,\n"
11348                 "\t\t\t\t.offset = %u,\n"
11349                 "\t\t\t\t.n_bytes = %u,\n"
11350                 "\t\t\t},\n"
11351                 "\t\t},\n"
11352                 "\t},\n",
11353                 instr_type_to_name(instr),
11354                 instr->hash_func.hash_func_id,
11355                 instr->hash_func.dst.offset,
11356                 instr->hash_func.dst.n_bits,
11357                 instr->hash_func.src.struct_id,
11358                 instr->hash_func.src.offset,
11359                 instr->hash_func.src.n_bytes);
11360 }
11361
11362 static void
11363 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11364 {
11365         int prefetch  = 0, idx_imm = 0, src_imm = 0;
11366
11367         if (instr->type == INSTR_REGPREFETCH_RH ||
11368             instr->type == INSTR_REGPREFETCH_RM ||
11369             instr->type == INSTR_REGPREFETCH_RI)
11370                 prefetch = 1;
11371
11372         /* index is the 3rd operand for the regrd instruction and the 2nd
11373          * operand for the regwr and regadd instructions.
11374          */
11375         if (instr->type == INSTR_REGPREFETCH_RI ||
11376             instr->type == INSTR_REGRD_HRI ||
11377             instr->type == INSTR_REGRD_MRI ||
11378             instr->type == INSTR_REGWR_RIH ||
11379             instr->type == INSTR_REGWR_RIM ||
11380             instr->type == INSTR_REGWR_RII ||
11381             instr->type == INSTR_REGADD_RIH ||
11382             instr->type == INSTR_REGADD_RIM ||
11383             instr->type == INSTR_REGADD_RII)
11384                 idx_imm = 1;
11385
11386         /* src is the 3rd operand for the regwr and regadd instructions. */
11387         if (instr->type == INSTR_REGWR_RHI ||
11388             instr->type == INSTR_REGWR_RMI ||
11389             instr->type == INSTR_REGWR_RII ||
11390             instr->type == INSTR_REGADD_RHI ||
11391             instr->type == INSTR_REGADD_RMI ||
11392             instr->type == INSTR_REGADD_RII)
11393                 src_imm = 1;
11394
11395         /* instr.regarray.regarray_id. */
11396         fprintf(f,
11397                 "\t{\n"
11398                 "\t\t.type = %s,\n"
11399                 "\t\t.regarray = {\n"
11400                 "\t\t\t.regarray_id = %u,\n",
11401                 instr_type_to_name(instr),
11402                 instr->regarray.regarray_id);
11403
11404         /* instr.regarray.idx / instr.regarray.idx_val. */
11405         if (!idx_imm)
11406                 fprintf(f,
11407                         "\t\t\t\t.idx = {\n"
11408                         "\t\t\t\t\t.struct_id = %u,\n"
11409                         "\t\t\t\t\t.n_bits = %u,\n"
11410                         "\t\t\t\t\t.offset = %u,\n"
11411                         "\t\t\t\t},\n",
11412                         instr->regarray.idx.struct_id,
11413                         instr->regarray.idx.n_bits,
11414                         instr->regarray.idx.offset);
11415         else
11416                 fprintf(f,
11417                         "\t\t\t\t.idx_val = %u,\n",
11418                         instr->regarray.idx_val);
11419
11420         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
11421         if (!prefetch) {
11422                 if (!src_imm)
11423                         fprintf(f,
11424                                 "\t\t\t\t.dstsrc = {\n"
11425                                 "\t\t\t\t\t.struct_id = %u,\n"
11426                                 "\t\t\t\t\t.n_bits = %u,\n"
11427                                 "\t\t\t\t\t.offset = %u,\n"
11428                                 "\t\t\t\t},\n",
11429                                 instr->regarray.dstsrc.struct_id,
11430                                 instr->regarray.dstsrc.n_bits,
11431                                 instr->regarray.dstsrc.offset);
11432                 else
11433                         fprintf(f,
11434                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
11435                                 instr->regarray.dstsrc_val);
11436         }
11437
11438         /* instr.regarray and instr - closing curly braces. */
11439         fprintf(f,
11440                 "\t\t},\n"
11441                 "\t},\n");
11442 }
11443
11444 static void
11445 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11446 {
11447         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
11448
11449         if (instr->type == INSTR_METPREFETCH_H ||
11450             instr->type == INSTR_METPREFETCH_M ||
11451             instr->type == INSTR_METPREFETCH_I)
11452                 prefetch = 1;
11453
11454         /* idx_imm. */
11455         if (instr->type == INSTR_METPREFETCH_I ||
11456             instr->type == INSTR_METER_IHM ||
11457             instr->type == INSTR_METER_IHI ||
11458             instr->type == INSTR_METER_IMM ||
11459             instr->type == INSTR_METER_IMI)
11460                 idx_imm = 1;
11461
11462         /* color_in_imm. */
11463         if (instr->type == INSTR_METER_HHI ||
11464             instr->type == INSTR_METER_HMI ||
11465             instr->type == INSTR_METER_MHI ||
11466             instr->type == INSTR_METER_MMI ||
11467             instr->type == INSTR_METER_IHI ||
11468             instr->type == INSTR_METER_IMI)
11469                 color_in_imm = 1;
11470
11471         /* instr.meter.metarray_id. */
11472         fprintf(f,
11473                 "\t{\n"
11474                 "\t\t.type = %s,\n"
11475                 "\t\t.meter = {\n"
11476                 "\t\t\t.metarray_id = %u,\n",
11477                 instr_type_to_name(instr),
11478                 instr->meter.metarray_id);
11479
11480         /* instr.meter.idx / instr.meter.idx_val. */
11481         if (!idx_imm)
11482                 fprintf(f,
11483                         "\t\t\t.idx = {\n"
11484                         "\t\t\t\t.struct_id = %u,\n"
11485                         "\t\t\t\t.n_bits = %u,\n"
11486                         "\t\t\t\t.offset = %u,\n"
11487                         "\t\t\t},\n",
11488                         instr->meter.idx.struct_id,
11489                         instr->meter.idx.n_bits,
11490                         instr->meter.idx.offset);
11491         else
11492                 fprintf(f,
11493                         "\t\t\t.idx_val = %u,\n",
11494                         instr->meter.idx_val);
11495
11496         if (!prefetch) {
11497                 /* instr.meter.length. */
11498                 fprintf(f,
11499                         "\t\t\t.length = {\n"
11500                         "\t\t\t\t.struct_id = %u,\n"
11501                         "\t\t\t\t.n_bits = %u,\n"
11502                         "\t\t\t\t.offset = %u,\n"
11503                         "\t\t\t},\n",
11504                         instr->meter.length.struct_id,
11505                         instr->meter.length.n_bits,
11506                         instr->meter.length.offset);
11507
11508                 /* instr.meter.color_in / instr.meter.color_in_val. */
11509                 if (!color_in_imm)
11510                         fprintf(f,
11511                                 "\t\t\t.color_in = {\n"
11512                                 "\t\t\t\t.struct_id = %u,\n"
11513                                 "\t\t\t\t.n_bits = %u,\n"
11514                                 "\t\t\t\t.offset = %u,\n"
11515                                 "\t\t\t},\n",
11516                                 instr->meter.color_in.struct_id,
11517                                 instr->meter.color_in.n_bits,
11518                                 instr->meter.color_in.offset);
11519                 else
11520                         fprintf(f,
11521                                 "\t\t\t.color_in_val = %u,\n",
11522                                 (uint32_t)instr->meter.color_in_val);
11523
11524                 /* instr.meter.color_out. */
11525                 fprintf(f,
11526                         "\t\t\t.color_out = {\n"
11527                         "\t\t\t\t.struct_id = %u,\n"
11528                         "\t\t\t\t.n_bits = %u,\n"
11529                         "\t\t\t\t.offset = %u,\n"
11530                         "\t\t\t},\n",
11531                         instr->meter.color_out.struct_id,
11532                         instr->meter.color_out.n_bits,
11533                         instr->meter.color_out.offset);
11534         }
11535
11536         /* instr.meter and instr - closing curly braces. */
11537         fprintf(f,
11538                 "\t\t},\n"
11539                 "\t},\n");
11540 }
11541
11542 static void
11543 instr_table_export(struct instruction *instr,
11544                 FILE *f)
11545 {
11546         fprintf(f,
11547                 "\t{\n"
11548                 "\t\t.type = %s,\n"
11549                 "\t\t.table = {\n"
11550                 "\t\t\t.table_id = %u,\n"
11551                 "\t\t},\n"
11552                 "\t},\n",
11553                 instr_type_to_name(instr),
11554                 instr->table.table_id);
11555 }
11556
11557 static void
11558 instr_learn_export(struct instruction *instr, FILE *f)
11559 {
11560         fprintf(f,
11561                 "\t{\n"
11562                 "\t\t.type = %s,\n"
11563                 "\t\t.learn = {\n"
11564                 "\t\t\t.action_id = %u,\n"
11565                 "\t\t\t.mf_first_arg_offset = %u,\n"
11566                 "\t\t\t.mf_timeout_id_offset = %u,\n"
11567                 "\t\t\t.mf_timeout_id_n_bits = %u,\n"
11568                 "\t\t},\n"
11569                 "\t},\n",
11570                 instr_type_to_name(instr),
11571                 instr->learn.action_id,
11572                 instr->learn.mf_first_arg_offset,
11573                 instr->learn.mf_timeout_id_offset,
11574                 instr->learn.mf_timeout_id_n_bits);
11575 }
11576
11577 static void
11578 instr_rearm_export(struct instruction *instr, FILE *f)
11579 {
11580         if (instr->type == INSTR_LEARNER_REARM)
11581                 fprintf(f,
11582                         "\t{\n"
11583                         "\t\t.type = %s,\n"
11584                         "\t},\n",
11585                         instr_type_to_name(instr));
11586         else
11587                 fprintf(f,
11588                         "\t{\n"
11589                         "\t\t.type = %s,\n"
11590                         "\t\t.learn = {\n"
11591                         "\t\t\t.mf_timeout_id_offset = %u,\n"
11592                         "\t\t\t.mf_timeout_id_n_bits = %u,\n"
11593                         "\t\t},\n"
11594                         "\t},\n",
11595                         instr_type_to_name(instr),
11596                         instr->learn.mf_timeout_id_offset,
11597                         instr->learn.mf_timeout_id_n_bits);
11598 }
11599
11600 static void
11601 instr_forget_export(struct instruction *instr, FILE *f)
11602 {
11603         fprintf(f,
11604                 "\t{\n"
11605                 "\t\t.type = %s,\n"
11606                 "\t},\n",
11607                 instr_type_to_name(instr));
11608 }
11609
11610 static void
11611 instr_extern_export(struct instruction *instr, FILE *f)
11612 {
11613         if (instr->type == INSTR_EXTERN_OBJ)
11614                 fprintf(f,
11615                         "\t{\n"
11616                         "\t\t.type = %s,\n"
11617                         "\t\t.ext_obj = {\n"
11618                         "\t\t\t.ext_obj_id = %u,\n"
11619                         "\t\t\t.func_id = %u,\n"
11620                         "\t\t},\n"
11621                         "\t},\n",
11622                         instr_type_to_name(instr),
11623                         instr->ext_obj.ext_obj_id,
11624                         instr->ext_obj.func_id);
11625         else
11626                 fprintf(f,
11627                         "\t{\n"
11628                         "\t\t.type = %s,\n"
11629                         "\t\t.ext_func = {\n"
11630                         "\t\t\t.ext_func_id = %u,\n"
11631                         "\t\t},\n"
11632                         "\t},\n",
11633                         instr_type_to_name(instr),
11634                         instr->ext_func.ext_func_id);
11635 }
11636
11637 static void
11638 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
11639 {
11640         fprintf(f,
11641                 "\t{\n"
11642                 "\t\t.type = %s,\n"
11643                 "\t\t.jmp = {\n"
11644                 "\t\t\t.ip = NULL,\n",
11645                 instr_type_to_name(instr));
11646
11647         switch (instr->type) {
11648         case INSTR_JMP_VALID:
11649         case INSTR_JMP_INVALID:
11650                 fprintf(f,
11651                         "\t\t\t.header_id = %u,\n",
11652                         instr->jmp.header_id);
11653                 break;
11654
11655         case INSTR_JMP_ACTION_HIT:
11656         case INSTR_JMP_ACTION_MISS:
11657                 fprintf(f,
11658                         "\t\t\t.action_id = %u,\n",
11659                         instr->jmp.action_id);
11660                 break;
11661
11662         case INSTR_JMP_EQ:
11663         case INSTR_JMP_EQ_MH:
11664         case INSTR_JMP_EQ_HM:
11665         case INSTR_JMP_EQ_HH:
11666         case INSTR_JMP_NEQ:
11667         case INSTR_JMP_NEQ_MH:
11668         case INSTR_JMP_NEQ_HM:
11669         case INSTR_JMP_NEQ_HH:
11670         case INSTR_JMP_LT:
11671         case INSTR_JMP_LT_MH:
11672         case INSTR_JMP_LT_HM:
11673         case INSTR_JMP_LT_HH:
11674         case INSTR_JMP_GT:
11675         case INSTR_JMP_GT_MH:
11676         case INSTR_JMP_GT_HM:
11677         case INSTR_JMP_GT_HH:
11678                 fprintf(f,
11679                         "\t\t\t.a = {\n"
11680                         "\t\t\t\t.struct_id = %u,\n"
11681                         "\t\t\t\t.n_bits = %u,\n"
11682                         "\t\t\t\t.offset = %u,\n"
11683                         "\t\t\t},\n"
11684                         "\t\t\t.b = {\n"
11685                         "\t\t\t\t.struct_id = %u,\n"
11686                         "\t\t\t\t.n_bits = %u,\n"
11687                         "\t\t\t\t.offset = %u,\n"
11688                         "\t\t\t},\n",
11689                         instr->jmp.a.struct_id,
11690                         instr->jmp.a.n_bits,
11691                         instr->jmp.a.offset,
11692                         instr->jmp.b.struct_id,
11693                         instr->jmp.b.n_bits,
11694                         instr->jmp.b.offset);
11695                 break;
11696
11697         case INSTR_JMP_EQ_I:
11698         case INSTR_JMP_NEQ_I:
11699         case INSTR_JMP_LT_MI:
11700         case INSTR_JMP_LT_HI:
11701         case INSTR_JMP_GT_MI:
11702         case INSTR_JMP_GT_HI:
11703                 fprintf(f,
11704                         "\t\t\t.a = {\n"
11705                         "\t\t\t\t.struct_id = %u,\n"
11706                         "\t\t\t\t.n_bits = %u,\n"
11707                         "\t\t\t\t.offset = %u,\n"
11708                         "\t\t\t}\n,"
11709                         "\t\t\t.b_val = %" PRIu64 ",\n",
11710                         instr->jmp.a.struct_id,
11711                         instr->jmp.a.n_bits,
11712                         instr->jmp.a.offset,
11713                         instr->jmp.b_val);
11714                 break;
11715
11716         default:
11717                 break;
11718         }
11719
11720         fprintf(f,
11721                 "\t\t},\n"
11722                 "\t},\n");
11723 }
11724
11725 static void
11726 instr_return_export(struct instruction *instr,
11727                 FILE *f)
11728 {
11729         fprintf(f,
11730                 "\t{\n"
11731                 "\t\t.type = %s,\n",
11732                 instr_type_to_name(instr));
11733
11734         fprintf(f,
11735                 "\t},\n");
11736 }
11737
11738 static instruction_export_t export_table[] = {
11739         [INSTR_RX] = instr_io_export,
11740
11741         [INSTR_TX] = instr_io_export,
11742         [INSTR_TX_I] = instr_io_export,
11743         [INSTR_DROP] = instr_io_export,
11744         [INSTR_MIRROR] = instr_mirror_export,
11745         [INSTR_RECIRCULATE] = instr_recirculate_export,
11746         [INSTR_RECIRCID] = instr_recircid_export,
11747
11748         [INSTR_HDR_EXTRACT] = instr_io_export,
11749         [INSTR_HDR_EXTRACT2] = instr_io_export,
11750         [INSTR_HDR_EXTRACT3] = instr_io_export,
11751         [INSTR_HDR_EXTRACT4] = instr_io_export,
11752         [INSTR_HDR_EXTRACT5] = instr_io_export,
11753         [INSTR_HDR_EXTRACT6] = instr_io_export,
11754         [INSTR_HDR_EXTRACT7] = instr_io_export,
11755         [INSTR_HDR_EXTRACT8] = instr_io_export,
11756
11757         [INSTR_HDR_EXTRACT_M] = instr_io_export,
11758
11759         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
11760
11761         [INSTR_HDR_EMIT] = instr_io_export,
11762         [INSTR_HDR_EMIT_TX] = instr_io_export,
11763         [INSTR_HDR_EMIT2_TX] = instr_io_export,
11764         [INSTR_HDR_EMIT3_TX] = instr_io_export,
11765         [INSTR_HDR_EMIT4_TX] = instr_io_export,
11766         [INSTR_HDR_EMIT5_TX] = instr_io_export,
11767         [INSTR_HDR_EMIT6_TX] = instr_io_export,
11768         [INSTR_HDR_EMIT7_TX] = instr_io_export,
11769         [INSTR_HDR_EMIT8_TX] = instr_io_export,
11770
11771         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
11772         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
11773
11774         [INSTR_MOV] = instr_mov_export,
11775         [INSTR_MOV_MH] = instr_mov_export,
11776         [INSTR_MOV_HM] = instr_mov_export,
11777         [INSTR_MOV_HH] = instr_mov_export,
11778         [INSTR_MOV_I] = instr_mov_export,
11779
11780         [INSTR_DMA_HT]  = instr_dma_ht_export,
11781         [INSTR_DMA_HT2] = instr_dma_ht_export,
11782         [INSTR_DMA_HT3] = instr_dma_ht_export,
11783         [INSTR_DMA_HT4] = instr_dma_ht_export,
11784         [INSTR_DMA_HT5] = instr_dma_ht_export,
11785         [INSTR_DMA_HT6] = instr_dma_ht_export,
11786         [INSTR_DMA_HT7] = instr_dma_ht_export,
11787         [INSTR_DMA_HT8] = instr_dma_ht_export,
11788
11789         [INSTR_ALU_ADD] = instr_alu_export,
11790         [INSTR_ALU_ADD_MH] = instr_alu_export,
11791         [INSTR_ALU_ADD_HM] = instr_alu_export,
11792         [INSTR_ALU_ADD_HH] = instr_alu_export,
11793         [INSTR_ALU_ADD_MI] = instr_alu_export,
11794         [INSTR_ALU_ADD_HI] = instr_alu_export,
11795
11796         [INSTR_ALU_SUB] = instr_alu_export,
11797         [INSTR_ALU_SUB_MH] = instr_alu_export,
11798         [INSTR_ALU_SUB_HM] = instr_alu_export,
11799         [INSTR_ALU_SUB_HH] = instr_alu_export,
11800         [INSTR_ALU_SUB_MI] = instr_alu_export,
11801         [INSTR_ALU_SUB_HI] = instr_alu_export,
11802
11803         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
11804         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
11805         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
11806         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
11807
11808         [INSTR_ALU_AND] = instr_alu_export,
11809         [INSTR_ALU_AND_MH] = instr_alu_export,
11810         [INSTR_ALU_AND_HM] = instr_alu_export,
11811         [INSTR_ALU_AND_HH] = instr_alu_export,
11812         [INSTR_ALU_AND_I] = instr_alu_export,
11813
11814         [INSTR_ALU_OR] = instr_alu_export,
11815         [INSTR_ALU_OR_MH] = instr_alu_export,
11816         [INSTR_ALU_OR_HM] = instr_alu_export,
11817         [INSTR_ALU_OR_HH] = instr_alu_export,
11818         [INSTR_ALU_OR_I] = instr_alu_export,
11819
11820         [INSTR_ALU_XOR] = instr_alu_export,
11821         [INSTR_ALU_XOR_MH] = instr_alu_export,
11822         [INSTR_ALU_XOR_HM] = instr_alu_export,
11823         [INSTR_ALU_XOR_HH] = instr_alu_export,
11824         [INSTR_ALU_XOR_I] = instr_alu_export,
11825
11826         [INSTR_ALU_SHL] = instr_alu_export,
11827         [INSTR_ALU_SHL_MH] = instr_alu_export,
11828         [INSTR_ALU_SHL_HM] = instr_alu_export,
11829         [INSTR_ALU_SHL_HH] = instr_alu_export,
11830         [INSTR_ALU_SHL_MI] = instr_alu_export,
11831         [INSTR_ALU_SHL_HI] = instr_alu_export,
11832
11833         [INSTR_ALU_SHR] = instr_alu_export,
11834         [INSTR_ALU_SHR_MH] = instr_alu_export,
11835         [INSTR_ALU_SHR_HM] = instr_alu_export,
11836         [INSTR_ALU_SHR_HH] = instr_alu_export,
11837         [INSTR_ALU_SHR_MI] = instr_alu_export,
11838         [INSTR_ALU_SHR_HI] = instr_alu_export,
11839
11840         [INSTR_REGPREFETCH_RH] = instr_reg_export,
11841         [INSTR_REGPREFETCH_RM] = instr_reg_export,
11842         [INSTR_REGPREFETCH_RI] = instr_reg_export,
11843
11844         [INSTR_REGRD_HRH] = instr_reg_export,
11845         [INSTR_REGRD_HRM] = instr_reg_export,
11846         [INSTR_REGRD_MRH] = instr_reg_export,
11847         [INSTR_REGRD_MRM] = instr_reg_export,
11848         [INSTR_REGRD_HRI] = instr_reg_export,
11849         [INSTR_REGRD_MRI] = instr_reg_export,
11850
11851         [INSTR_REGWR_RHH] = instr_reg_export,
11852         [INSTR_REGWR_RHM] = instr_reg_export,
11853         [INSTR_REGWR_RMH] = instr_reg_export,
11854         [INSTR_REGWR_RMM] = instr_reg_export,
11855         [INSTR_REGWR_RHI] = instr_reg_export,
11856         [INSTR_REGWR_RMI] = instr_reg_export,
11857         [INSTR_REGWR_RIH] = instr_reg_export,
11858         [INSTR_REGWR_RIM] = instr_reg_export,
11859         [INSTR_REGWR_RII] = instr_reg_export,
11860
11861         [INSTR_REGADD_RHH] = instr_reg_export,
11862         [INSTR_REGADD_RHM] = instr_reg_export,
11863         [INSTR_REGADD_RMH] = instr_reg_export,
11864         [INSTR_REGADD_RMM] = instr_reg_export,
11865         [INSTR_REGADD_RHI] = instr_reg_export,
11866         [INSTR_REGADD_RMI] = instr_reg_export,
11867         [INSTR_REGADD_RIH] = instr_reg_export,
11868         [INSTR_REGADD_RIM] = instr_reg_export,
11869         [INSTR_REGADD_RII] = instr_reg_export,
11870
11871         [INSTR_METPREFETCH_H] = instr_meter_export,
11872         [INSTR_METPREFETCH_M] = instr_meter_export,
11873         [INSTR_METPREFETCH_I] = instr_meter_export,
11874
11875         [INSTR_METER_HHM] = instr_meter_export,
11876         [INSTR_METER_HHI] = instr_meter_export,
11877         [INSTR_METER_HMM] = instr_meter_export,
11878         [INSTR_METER_HMI] = instr_meter_export,
11879         [INSTR_METER_MHM] = instr_meter_export,
11880         [INSTR_METER_MHI] = instr_meter_export,
11881         [INSTR_METER_MMM] = instr_meter_export,
11882         [INSTR_METER_MMI] = instr_meter_export,
11883         [INSTR_METER_IHM] = instr_meter_export,
11884         [INSTR_METER_IHI] = instr_meter_export,
11885         [INSTR_METER_IMM] = instr_meter_export,
11886         [INSTR_METER_IMI] = instr_meter_export,
11887
11888         [INSTR_TABLE] = instr_table_export,
11889         [INSTR_TABLE_AF] = instr_table_export,
11890         [INSTR_SELECTOR] = instr_table_export,
11891         [INSTR_LEARNER] = instr_table_export,
11892         [INSTR_LEARNER_AF] = instr_table_export,
11893
11894         [INSTR_LEARNER_LEARN] = instr_learn_export,
11895         [INSTR_LEARNER_REARM] = instr_rearm_export,
11896         [INSTR_LEARNER_REARM_NEW] = instr_rearm_export,
11897         [INSTR_LEARNER_FORGET] = instr_forget_export,
11898
11899         [INSTR_EXTERN_OBJ] = instr_extern_export,
11900         [INSTR_EXTERN_FUNC] = instr_extern_export,
11901         [INSTR_HASH_FUNC] = instr_hash_export,
11902
11903         [INSTR_JMP] = instr_jmp_export,
11904         [INSTR_JMP_VALID] = instr_jmp_export,
11905         [INSTR_JMP_INVALID] = instr_jmp_export,
11906         [INSTR_JMP_HIT] = instr_jmp_export,
11907         [INSTR_JMP_MISS] = instr_jmp_export,
11908         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11909         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11910
11911         [INSTR_JMP_EQ] = instr_jmp_export,
11912         [INSTR_JMP_EQ_MH] = instr_jmp_export,
11913         [INSTR_JMP_EQ_HM] = instr_jmp_export,
11914         [INSTR_JMP_EQ_HH] = instr_jmp_export,
11915         [INSTR_JMP_EQ_I] = instr_jmp_export,
11916
11917         [INSTR_JMP_NEQ] = instr_jmp_export,
11918         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11919         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11920         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11921         [INSTR_JMP_NEQ_I] = instr_jmp_export,
11922
11923         [INSTR_JMP_LT] = instr_jmp_export,
11924         [INSTR_JMP_LT_MH] = instr_jmp_export,
11925         [INSTR_JMP_LT_HM] = instr_jmp_export,
11926         [INSTR_JMP_LT_HH] = instr_jmp_export,
11927         [INSTR_JMP_LT_MI] = instr_jmp_export,
11928         [INSTR_JMP_LT_HI] = instr_jmp_export,
11929
11930         [INSTR_JMP_GT] = instr_jmp_export,
11931         [INSTR_JMP_GT_MH] = instr_jmp_export,
11932         [INSTR_JMP_GT_HM] = instr_jmp_export,
11933         [INSTR_JMP_GT_HH] = instr_jmp_export,
11934         [INSTR_JMP_GT_MI] = instr_jmp_export,
11935         [INSTR_JMP_GT_HI] = instr_jmp_export,
11936
11937         [INSTR_RETURN] = instr_return_export,
11938 };
11939
11940 static void
11941 action_data_codegen(struct action *a, FILE *f)
11942 {
11943         uint32_t i;
11944
11945         fprintf(f,
11946                 "static const struct instruction action_%s_instructions[] = {\n",
11947                 a->name);
11948
11949         for (i = 0; i < a->n_instructions; i++) {
11950                 struct instruction *instr = &a->instructions[i];
11951                 instruction_export_t func = export_table[instr->type];
11952
11953                 func(instr, f);
11954         }
11955
11956         fprintf(f, "};\n");
11957 }
11958
11959 static const char *
11960 instr_type_to_func(struct instruction *instr)
11961 {
11962         switch (instr->type) {
11963         case INSTR_RX: return NULL;
11964
11965         case INSTR_TX: return "__instr_tx_exec";
11966         case INSTR_TX_I: return "__instr_tx_i_exec";
11967         case INSTR_DROP: return "__instr_drop_exec";
11968         case INSTR_MIRROR: return "__instr_mirror_exec";
11969         case INSTR_RECIRCULATE: return "__instr_recirculate_exec";
11970         case INSTR_RECIRCID: return "__instr_recircid_exec";
11971
11972         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11973         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11974         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11975         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11976         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11977         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11978         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11979         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11980
11981         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11982
11983         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11984
11985         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11986         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11987         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11988         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11989         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11990         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11991         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11992         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11993         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11994
11995         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11996         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11997
11998         case INSTR_MOV: return "__instr_mov_exec";
11999         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
12000         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
12001         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
12002         case INSTR_MOV_I: return "__instr_mov_i_exec";
12003
12004         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
12005         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
12006         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
12007         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
12008         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
12009         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
12010         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
12011         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
12012
12013         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
12014         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
12015         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
12016         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
12017         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
12018         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
12019
12020         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
12021         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
12022         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
12023         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
12024         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
12025         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
12026
12027         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
12028         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
12029         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
12030         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
12031
12032         case INSTR_ALU_AND: return "__instr_alu_and_exec";
12033         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
12034         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
12035         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
12036         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
12037
12038         case INSTR_ALU_OR: return "__instr_alu_or_exec";
12039         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
12040         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
12041         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
12042         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
12043
12044         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
12045         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
12046         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
12047         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
12048         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
12049
12050         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
12051         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
12052         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
12053         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
12054         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
12055         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
12056
12057         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
12058         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
12059         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
12060         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
12061         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
12062         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
12063
12064         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
12065         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
12066         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
12067
12068         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
12069         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
12070         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
12071         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
12072         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
12073         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
12074
12075         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
12076         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
12077         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
12078         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
12079         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
12080         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
12081         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
12082         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
12083         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
12084
12085         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
12086         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
12087         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
12088         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
12089         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
12090         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
12091         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
12092         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
12093         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
12094
12095         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
12096         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
12097         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
12098
12099         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
12100         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
12101         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
12102         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
12103         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
12104         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
12105         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
12106         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
12107         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
12108         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
12109         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
12110         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
12111
12112         case INSTR_TABLE: return NULL;
12113         case INSTR_TABLE_AF: return NULL;
12114         case INSTR_SELECTOR: return NULL;
12115         case INSTR_LEARNER: return NULL;
12116         case INSTR_LEARNER_AF: return NULL;
12117
12118         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
12119         case INSTR_LEARNER_REARM: return "__instr_rearm_exec";
12120         case INSTR_LEARNER_REARM_NEW: return "__instr_rearm_new_exec";
12121         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
12122
12123         case INSTR_EXTERN_OBJ: return NULL;
12124         case INSTR_EXTERN_FUNC: return NULL;
12125         case INSTR_HASH_FUNC: return "__instr_hash_func_exec";
12126
12127         case INSTR_JMP: return NULL;
12128         case INSTR_JMP_VALID: return NULL;
12129         case INSTR_JMP_INVALID: return NULL;
12130         case INSTR_JMP_HIT: return NULL;
12131         case INSTR_JMP_MISS: return NULL;
12132         case INSTR_JMP_ACTION_HIT: return NULL;
12133         case INSTR_JMP_ACTION_MISS: return NULL;
12134         case INSTR_JMP_EQ: return NULL;
12135         case INSTR_JMP_EQ_MH: return NULL;
12136         case INSTR_JMP_EQ_HM: return NULL;
12137         case INSTR_JMP_EQ_HH: return NULL;
12138         case INSTR_JMP_EQ_I: return NULL;
12139         case INSTR_JMP_NEQ: return NULL;
12140         case INSTR_JMP_NEQ_MH: return NULL;
12141         case INSTR_JMP_NEQ_HM: return NULL;
12142         case INSTR_JMP_NEQ_HH: return NULL;
12143         case INSTR_JMP_NEQ_I: return NULL;
12144         case INSTR_JMP_LT: return NULL;
12145         case INSTR_JMP_LT_MH: return NULL;
12146         case INSTR_JMP_LT_HM: return NULL;
12147         case INSTR_JMP_LT_HH: return NULL;
12148         case INSTR_JMP_LT_MI: return NULL;
12149         case INSTR_JMP_LT_HI: return NULL;
12150         case INSTR_JMP_GT: return NULL;
12151         case INSTR_JMP_GT_MH: return NULL;
12152         case INSTR_JMP_GT_HM: return NULL;
12153         case INSTR_JMP_GT_HH: return NULL;
12154         case INSTR_JMP_GT_MI: return NULL;
12155         case INSTR_JMP_GT_HI: return NULL;
12156
12157         case INSTR_RETURN: return NULL;
12158
12159         default: return NULL;
12160         }
12161 }
12162
12163 static void
12164 action_instr_does_tx_codegen(struct action *a,
12165                         uint32_t instr_pos,
12166                         struct instruction *instr,
12167                         FILE *f)
12168 {
12169         fprintf(f,
12170                 "%s(p, t, &action_%s_instructions[%u]);\n"
12171                 "\tthread_ip_reset(p, t);\n"
12172                 "\tinstr_rx_exec(p);\n"
12173                 "\treturn;\n",
12174                 instr_type_to_func(instr),
12175                 a->name,
12176                 instr_pos);
12177 }
12178
12179 static void
12180 action_instr_extern_obj_codegen(struct action *a,
12181                                 uint32_t instr_pos,
12182                                 FILE *f)
12183 {
12184         fprintf(f,
12185                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
12186                 a->name,
12187                 instr_pos);
12188 }
12189
12190 static void
12191 action_instr_extern_func_codegen(struct action *a,
12192                                  uint32_t instr_pos,
12193                                  FILE *f)
12194 {
12195         fprintf(f,
12196                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
12197                 a->name,
12198                 instr_pos);
12199 }
12200
12201 static void
12202 action_instr_jmp_codegen(struct action *a,
12203                          uint32_t instr_pos,
12204                          struct instruction *instr,
12205                          struct instruction_data *data,
12206                          FILE *f)
12207 {
12208         switch (instr->type) {
12209         case INSTR_JMP:
12210                 fprintf(f,
12211                         "goto %s;\n",
12212                         data->jmp_label);
12213                 return;
12214
12215         case INSTR_JMP_VALID:
12216                 fprintf(f,
12217                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
12218                         "\t\tgoto %s;\n",
12219                         a->name,
12220                         instr_pos,
12221                         data->jmp_label);
12222                 return;
12223
12224         case INSTR_JMP_INVALID:
12225                 fprintf(f,
12226                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
12227                         "\t\tgoto %s;\n",
12228                         a->name,
12229                         instr_pos,
12230                         data->jmp_label);
12231                 return;
12232
12233         case INSTR_JMP_HIT:
12234                 fprintf(f,
12235                         "if (t->hit)\n"
12236                         "\t\tgoto %s;\n",
12237                         data->jmp_label);
12238                 return;
12239
12240         case INSTR_JMP_MISS:
12241                 fprintf(f,
12242                         "if (!t->hit)\n"
12243                         "\t\tgoto %s;\n",
12244                         data->jmp_label);
12245                 return;
12246
12247         case INSTR_JMP_ACTION_HIT:
12248                 fprintf(f,
12249                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
12250                         "\t\tgoto %s;\n",
12251                         a->name,
12252                         instr_pos,
12253                         data->jmp_label);
12254                 return;
12255
12256         case INSTR_JMP_ACTION_MISS:
12257                 fprintf(f,
12258                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
12259                         "\t\tgoto %s;\n",
12260                         a->name,
12261                         instr_pos,
12262                         data->jmp_label);
12263                 return;
12264
12265         case INSTR_JMP_EQ:
12266                 fprintf(f,
12267                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12268                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12269                         "\t\tgoto %s;\n",
12270                         a->name,
12271                         instr_pos,
12272                         a->name,
12273                         instr_pos,
12274                         data->jmp_label);
12275                 return;
12276
12277         case INSTR_JMP_EQ_MH:
12278                 fprintf(f,
12279                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12280                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12281                         "\t\tgoto %s;\n",
12282                         a->name,
12283                         instr_pos,
12284                         a->name,
12285                         instr_pos,
12286                         data->jmp_label);
12287                 return;
12288
12289         case INSTR_JMP_EQ_HM:
12290                 fprintf(f,
12291                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
12292                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12293                         "\t\tgoto %s;\n",
12294                         a->name,
12295                         instr_pos,
12296                         a->name,
12297                         instr_pos,
12298                         data->jmp_label);
12299                 return;
12300
12301         case INSTR_JMP_EQ_HH:
12302                 fprintf(f,
12303                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
12304                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12305                         "\t\tgoto %s;\n",
12306                         a->name,
12307                         instr_pos,
12308                         a->name,
12309                         instr_pos,
12310                         data->jmp_label);
12311                 return;
12312
12313         case INSTR_JMP_EQ_I:
12314                 fprintf(f,
12315                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12316                         "action_%s_instructions[%u].jmp.b_val)\n"
12317                         "\t\tgoto %s;\n",
12318                         a->name,
12319                         instr_pos,
12320                         a->name,
12321                         instr_pos,
12322                         data->jmp_label);
12323                 return;
12324
12325         case INSTR_JMP_NEQ:
12326                 fprintf(f,
12327                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12328                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12329                         "\t\tgoto %s;\n",
12330                         a->name,
12331                         instr_pos,
12332                         a->name,
12333                         instr_pos,
12334                         data->jmp_label);
12335                 return;
12336
12337         case INSTR_JMP_NEQ_MH:
12338                 fprintf(f,
12339                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12340                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12341                         "\t\tgoto %s;\n",
12342                         a->name,
12343                         instr_pos,
12344                         a->name,
12345                         instr_pos,
12346                         data->jmp_label);
12347                 return;
12348
12349         case INSTR_JMP_NEQ_HM:
12350                 fprintf(f,
12351                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
12352                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12353                         "\t\tgoto %s;\n",
12354                         a->name,
12355                         instr_pos,
12356                         a->name,
12357                         instr_pos,
12358                         data->jmp_label);
12359                 return;
12360
12361         case INSTR_JMP_NEQ_HH:
12362                 fprintf(f,
12363                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
12364                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12365                         "\t\tgoto %s;\n",
12366                         a->name,
12367                         instr_pos,
12368                         a->name,
12369                         instr_pos,
12370                         data->jmp_label);
12371                 return;
12372
12373         case INSTR_JMP_NEQ_I:
12374                 fprintf(f,
12375                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12376                         "action_%s_instructions[%u].jmp.b_val)\n"
12377                         "\t\tgoto %s;\n",
12378                         a->name,
12379                         instr_pos,
12380                         a->name,
12381                         instr_pos,
12382                         data->jmp_label);
12383                 return;
12384
12385         case INSTR_JMP_LT:
12386                 fprintf(f,
12387                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12388                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12389                         "\t\tgoto %s;\n",
12390                         a->name,
12391                         instr_pos,
12392                         a->name,
12393                         instr_pos,
12394                         data->jmp_label);
12395                 return;
12396
12397         case INSTR_JMP_LT_MH:
12398                 fprintf(f,
12399                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12400                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12401                         "\t\tgoto %s;\n",
12402                         a->name,
12403                         instr_pos,
12404                         a->name,
12405                         instr_pos,
12406                         data->jmp_label);
12407                 return;
12408
12409         case INSTR_JMP_LT_HM:
12410                 fprintf(f,
12411                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12412                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12413                         "\t\tgoto %s;\n",
12414                         a->name,
12415                         instr_pos,
12416                         a->name,
12417                         instr_pos,
12418                         data->jmp_label);
12419                 return;
12420
12421         case INSTR_JMP_LT_HH:
12422                 fprintf(f,
12423                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12424                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12425                         "\t\tgoto %s;\n",
12426                         a->name,
12427                         instr_pos,
12428                         a->name,
12429                         instr_pos,
12430                         data->jmp_label);
12431                 return;
12432
12433         case INSTR_JMP_LT_MI:
12434                 fprintf(f,
12435                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12436                         "action_%s_instructions[%u].jmp.b_val)\n"
12437                         "\t\tgoto %s;\n",
12438                         a->name,
12439                         instr_pos,
12440                         a->name,
12441                         instr_pos,
12442                         data->jmp_label);
12443                 return;
12444
12445         case INSTR_JMP_LT_HI:
12446                 fprintf(f,
12447                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12448                         "action_%s_instructions[%u].jmp.b_val)\n"
12449                         "\t\tgoto %s;\n",
12450                         a->name,
12451                         instr_pos,
12452                         a->name,
12453                         instr_pos,
12454                         data->jmp_label);
12455                 return;
12456
12457         case INSTR_JMP_GT:
12458                 fprintf(f,
12459                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12460                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12461                         "\t\tgoto %s;\n",
12462                         a->name,
12463                         instr_pos,
12464                         a->name,
12465                         instr_pos,
12466                         data->jmp_label);
12467                 return;
12468
12469         case INSTR_JMP_GT_MH:
12470                 fprintf(f,
12471                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12472                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12473                         "\t\tgoto %s;\n",
12474                         a->name,
12475                         instr_pos,
12476                         a->name,
12477                         instr_pos,
12478                         data->jmp_label);
12479                 return;
12480
12481         case INSTR_JMP_GT_HM:
12482                 fprintf(f,
12483                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12484                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12485                         "\t\tgoto %s;\n",
12486                         a->name,
12487                         instr_pos,
12488                         a->name,
12489                         instr_pos,
12490                         data->jmp_label);
12491                 return;
12492
12493         case INSTR_JMP_GT_HH:
12494                 fprintf(f,
12495                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12496                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12497                         "\t\tgoto %s;\n",
12498                         a->name,
12499                         instr_pos,
12500                         a->name,
12501                         instr_pos,
12502                         data->jmp_label);
12503                 return;
12504
12505         case INSTR_JMP_GT_MI:
12506                 fprintf(f,
12507                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12508                         "action_%s_instructions[%u].jmp.b_val)\n"
12509                         "\t\tgoto %s;\n",
12510                         a->name,
12511                         instr_pos,
12512                         a->name,
12513                         instr_pos,
12514                         data->jmp_label);
12515                 return;
12516
12517         case INSTR_JMP_GT_HI:
12518                 fprintf(f,
12519                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12520                         "action_%s_instructions[%u].jmp.b_val)\n"
12521                         "\t\tgoto %s;\n",
12522                         a->name,
12523                         instr_pos,
12524                         a->name,
12525                         instr_pos,
12526                         data->jmp_label);
12527                 return;
12528
12529         default:
12530                 return;
12531         }
12532 }
12533
12534 static void
12535 action_instr_return_codegen(FILE *f)
12536 {
12537         fprintf(f,
12538                 "return;\n");
12539 }
12540
12541 static void
12542 action_instr_codegen(struct action *a, FILE *f)
12543 {
12544         uint32_t i;
12545
12546         fprintf(f,
12547                 "void\n"
12548                 "action_%s_run(struct rte_swx_pipeline *p)\n"
12549                 "{\n"
12550                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12551                 "\n",
12552                 a->name);
12553
12554         for (i = 0; i < a->n_instructions; i++) {
12555                 struct instruction *instr = &a->instructions[i];
12556                 struct instruction_data *data = &a->instruction_data[i];
12557
12558                 /* Label, if present. */
12559                 if (data->label[0])
12560                         fprintf(f, "\n%s : ", data->label);
12561                 else
12562                         fprintf(f, "\n\t");
12563
12564                 /* TX instruction type. */
12565                 if (instruction_does_tx(instr)) {
12566                         action_instr_does_tx_codegen(a, i, instr, f);
12567                         continue;
12568                 }
12569
12570                 /* Extern object/function instruction type. */
12571                 if (instr->type == INSTR_EXTERN_OBJ) {
12572                         action_instr_extern_obj_codegen(a, i, f);
12573                         continue;
12574                 }
12575
12576                 if (instr->type == INSTR_EXTERN_FUNC) {
12577                         action_instr_extern_func_codegen(a, i, f);
12578                         continue;
12579                 }
12580
12581                 /* Jump instruction type. */
12582                 if (instruction_is_jmp(instr)) {
12583                         action_instr_jmp_codegen(a, i, instr, data, f);
12584                         continue;
12585                 }
12586
12587                 /* Return instruction type. */
12588                 if (instr->type == INSTR_RETURN) {
12589                         action_instr_return_codegen(f);
12590                         continue;
12591                 }
12592
12593                 /* Any other instruction type. */
12594                 fprintf(f,
12595                         "%s(p, t, &action_%s_instructions[%u]);\n",
12596                         instr_type_to_func(instr),
12597                         a->name,
12598                         i);
12599         }
12600
12601         fprintf(f, "}\n\n");
12602 }
12603
12604 struct instruction_group {
12605         TAILQ_ENTRY(instruction_group) node;
12606
12607         uint32_t group_id;
12608
12609         uint32_t first_instr_id;
12610
12611         uint32_t last_instr_id;
12612
12613         instr_exec_t func;
12614 };
12615
12616 TAILQ_HEAD(instruction_group_list, instruction_group);
12617
12618 static struct instruction_group *
12619 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
12620 {
12621         struct instruction_group *g;
12622
12623         TAILQ_FOREACH(g, igl, node)
12624                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
12625                         return g;
12626
12627         return NULL;
12628 }
12629
12630 static void
12631 instruction_group_list_free(struct instruction_group_list *igl)
12632 {
12633         if (!igl)
12634                 return;
12635
12636         for ( ; ; ) {
12637                 struct instruction_group *g;
12638
12639                 g = TAILQ_FIRST(igl);
12640                 if (!g)
12641                         break;
12642
12643                 TAILQ_REMOVE(igl, g, node);
12644                 free(g);
12645         }
12646
12647         free(igl);
12648 }
12649
12650 static struct instruction_group_list *
12651 instruction_group_list_create(struct rte_swx_pipeline *p)
12652 {
12653         struct instruction_group_list *igl = NULL;
12654         struct instruction_group *g = NULL;
12655         uint32_t n_groups = 0, i;
12656
12657         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
12658                 goto error;
12659
12660         /* List init. */
12661         igl = calloc(1, sizeof(struct instruction_group_list));
12662         if (!igl)
12663                 goto error;
12664
12665         TAILQ_INIT(igl);
12666
12667         /* Allocate the first group. */
12668         g = calloc(1, sizeof(struct instruction_group));
12669         if (!g)
12670                 goto error;
12671
12672         /* Iteration 1: Separate the instructions into groups based on the thread yield
12673          * instructions. Do not worry about the jump instructions at this point.
12674          */
12675         for (i = 0; i < p->n_instructions; i++) {
12676                 struct instruction *instr = &p->instructions[i];
12677
12678                 /* Check for thread yield instructions. */
12679                 if (!instruction_does_thread_yield(instr))
12680                         continue;
12681
12682                 /* If the current group contains at least one instruction, then finalize it (with
12683                  * the previous instruction), add it to the list and allocate a new group (that
12684                  * starts with the current instruction).
12685                  */
12686                 if (i - g->first_instr_id) {
12687                         /* Finalize the group. */
12688                         g->last_instr_id = i - 1;
12689
12690                         /* Add the group to the list. Advance the number of groups. */
12691                         TAILQ_INSERT_TAIL(igl, g, node);
12692                         n_groups++;
12693
12694                         /* Allocate a new group. */
12695                         g = calloc(1, sizeof(struct instruction_group));
12696                         if (!g)
12697                                 goto error;
12698
12699                         /* Initialize the new group. */
12700                         g->group_id = n_groups;
12701                         g->first_instr_id = i;
12702                 }
12703
12704                 /* Finalize the current group (with the current instruction, therefore this group
12705                  * contains just the current thread yield instruction), add it to the list and
12706                  * allocate a new group (that starts with the next instruction).
12707                  */
12708
12709                 /* Finalize the group. */
12710                 g->last_instr_id = i;
12711
12712                 /* Add the group to the list. Advance the number of groups. */
12713                 TAILQ_INSERT_TAIL(igl, g, node);
12714                 n_groups++;
12715
12716                 /* Allocate a new group. */
12717                 g = calloc(1, sizeof(struct instruction_group));
12718                 if (!g)
12719                         goto error;
12720
12721                 /* Initialize the new group. */
12722                 g->group_id = n_groups;
12723                 g->first_instr_id = i + 1;
12724         }
12725
12726         /* Handle the last group. */
12727         if (i - g->first_instr_id) {
12728                 /* Finalize the group. */
12729                 g->last_instr_id = i - 1;
12730
12731                 /* Add the group to the list. Advance the number of groups. */
12732                 TAILQ_INSERT_TAIL(igl, g, node);
12733                 n_groups++;
12734         } else
12735                 free(g);
12736
12737         g = NULL;
12738
12739         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
12740          * the destination of a jump instruction located in a different group ("far jump"), then the
12741          * current group has to be split, so that the instruction representing the far jump
12742          * destination is at the start of its group.
12743          */
12744         for ( ; ; ) {
12745                 int is_modified = 0;
12746
12747                 for (i = 0; i < p->n_instructions; i++) {
12748                         struct instruction_data *data = &p->instruction_data[i];
12749                         struct instruction_group *g;
12750                         uint32_t j;
12751
12752                         /* Continue when the current instruction is not a jump destination. */
12753                         if (!data->n_users)
12754                                 continue;
12755
12756                         g = instruction_group_list_group_find(igl, i);
12757                         if (!g)
12758                                 goto error;
12759
12760                         /* Find out all the jump instructions with this destination. */
12761                         for (j = 0; j < p->n_instructions; j++) {
12762                                 struct instruction *jmp_instr = &p->instructions[j];
12763                                 struct instruction_data *jmp_data = &p->instruction_data[j];
12764                                 struct instruction_group *jmp_g, *new_g;
12765
12766                                 /* Continue when not a jump instruction. Even when jump instruction,
12767                                  * continue when the jump destination is not this instruction.
12768                                  */
12769                                 if (!instruction_is_jmp(jmp_instr) ||
12770                                     strcmp(jmp_data->jmp_label, data->label))
12771                                         continue;
12772
12773                                 jmp_g = instruction_group_list_group_find(igl, j);
12774                                 if (!jmp_g)
12775                                         goto error;
12776
12777                                 /* Continue when both the jump instruction and the jump destination
12778                                  * instruction are in the same group. Even when in different groups,
12779                                  * still continue if the jump destination instruction is already the
12780                                  * first instruction of its group.
12781                                  */
12782                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
12783                                         continue;
12784
12785                                 /* Split the group of the current jump destination instruction to
12786                                  * make this instruction the first instruction of a new group.
12787                                  */
12788                                 new_g = calloc(1, sizeof(struct instruction_group));
12789                                 if (!new_g)
12790                                         goto error;
12791
12792                                 new_g->group_id = n_groups;
12793                                 new_g->first_instr_id = i;
12794                                 new_g->last_instr_id = g->last_instr_id;
12795
12796                                 g->last_instr_id = i - 1;
12797
12798                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
12799                                 n_groups++;
12800                                 is_modified = 1;
12801
12802                                 /* The decision to split this group (to make the current instruction
12803                                  * the first instruction of a new group) is already taken and fully
12804                                  * implemented, so no need to search for more reasons to do it.
12805                                  */
12806                                 break;
12807                         }
12808                 }
12809
12810                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
12811                  * previously considered local (i.e. the jump destination is in the same group as
12812                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
12813                  * different group than the jump instruction). Wost case scenario: each instruction
12814                  * that is a jump destination ends up as the first instruction of its group.
12815                  */
12816                 if (!is_modified)
12817                         break;
12818         }
12819
12820         /* Re-assign the group IDs to be in incremental order. */
12821         i = 0;
12822         TAILQ_FOREACH(g, igl, node) {
12823                 g->group_id = i;
12824
12825                 i++;
12826         }
12827
12828         return igl;
12829
12830 error:
12831         instruction_group_list_free(igl);
12832
12833         free(g);
12834
12835         return NULL;
12836 }
12837
12838 static void
12839 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12840                                uint32_t instr_pos,
12841                                struct instruction *instr,
12842                                FILE *f)
12843 {
12844         fprintf(f,
12845                 "%s(p, t, &pipeline_instructions[%u]);\n"
12846                 "\tthread_ip_reset(p, t);\n"
12847                 "\tinstr_rx_exec(p);\n"
12848                 "\treturn;\n",
12849                 instr_type_to_func(instr),
12850                 instr_pos);
12851 }
12852
12853 static int
12854 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12855                            struct instruction_group_list *igl,
12856                            uint32_t jmp_instr_id,
12857                            struct instruction *jmp_instr,
12858                            struct instruction_data *jmp_data,
12859                            FILE *f)
12860 {
12861         struct instruction_group *jmp_g, *g;
12862         struct instruction_data *data;
12863         uint32_t instr_id;
12864
12865         switch (jmp_instr->type) {
12866         case INSTR_JMP:
12867                 break;
12868
12869         case INSTR_JMP_VALID:
12870                 fprintf(f,
12871                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12872                         jmp_instr_id);
12873                 break;
12874
12875         case INSTR_JMP_INVALID:
12876                 fprintf(f,
12877                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12878                         jmp_instr_id);
12879                 break;
12880
12881         case INSTR_JMP_HIT:
12882                 fprintf(f,
12883                         "if (t->hit)\n");
12884                 break;
12885
12886         case INSTR_JMP_MISS:
12887                 fprintf(f,
12888                         "if (!t->hit)\n");
12889                 break;
12890
12891         case INSTR_JMP_ACTION_HIT:
12892                 fprintf(f,
12893                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12894                         jmp_instr_id);
12895                 break;
12896
12897         case INSTR_JMP_ACTION_MISS:
12898                 fprintf(f,
12899                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12900                         jmp_instr_id);
12901                 break;
12902
12903         case INSTR_JMP_EQ:
12904                 fprintf(f,
12905                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12906                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12907                         jmp_instr_id,
12908                         jmp_instr_id);
12909                 break;
12910
12911         case INSTR_JMP_EQ_MH:
12912                 fprintf(f,
12913                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12914                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12915                         jmp_instr_id,
12916                         jmp_instr_id);
12917                 break;
12918
12919         case INSTR_JMP_EQ_HM:
12920                 fprintf(f,
12921                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12922                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12923                         jmp_instr_id,
12924                         jmp_instr_id);
12925                 break;
12926
12927         case INSTR_JMP_EQ_HH:
12928                 fprintf(f,
12929                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12930                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12931                         jmp_instr_id,
12932                         jmp_instr_id);
12933                 break;
12934
12935         case INSTR_JMP_EQ_I:
12936                 fprintf(f,
12937                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12938                         "pipeline_instructions[%u].jmp.b_val)",
12939                         jmp_instr_id,
12940                         jmp_instr_id);
12941                 break;
12942
12943         case INSTR_JMP_NEQ:
12944                 fprintf(f,
12945                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12946                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12947                         jmp_instr_id,
12948                         jmp_instr_id);
12949                 break;
12950
12951         case INSTR_JMP_NEQ_MH:
12952                 fprintf(f,
12953                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12954                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12955                         jmp_instr_id,
12956                         jmp_instr_id);
12957                 break;
12958
12959         case INSTR_JMP_NEQ_HM:
12960                 fprintf(f,
12961                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12962                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12963                         jmp_instr_id,
12964                         jmp_instr_id);
12965                 break;
12966
12967         case INSTR_JMP_NEQ_HH:
12968                 fprintf(f,
12969                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12970                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12971                         jmp_instr_id,
12972                         jmp_instr_id);
12973                 break;
12974
12975         case INSTR_JMP_NEQ_I:
12976                 fprintf(f,
12977                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12978                         "pipeline_instructions[%u].jmp.b_val)",
12979                         jmp_instr_id,
12980                         jmp_instr_id);
12981                 break;
12982
12983         case INSTR_JMP_LT:
12984                 fprintf(f,
12985                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12986                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12987                         jmp_instr_id,
12988                         jmp_instr_id);
12989                 break;
12990
12991         case INSTR_JMP_LT_MH:
12992                 fprintf(f,
12993                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12994                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12995                         jmp_instr_id,
12996                         jmp_instr_id);
12997                 break;
12998
12999         case INSTR_JMP_LT_HM:
13000                 fprintf(f,
13001                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
13002                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
13003                         jmp_instr_id,
13004                         jmp_instr_id);
13005                 break;
13006
13007         case INSTR_JMP_LT_HH:
13008                 fprintf(f,
13009                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
13010                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13011                         jmp_instr_id,
13012                         jmp_instr_id);
13013                 break;
13014
13015         case INSTR_JMP_LT_MI:
13016                 fprintf(f,
13017                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
13018                         "pipeline_instructions[%u].jmp.b_val)",
13019                         jmp_instr_id,
13020                         jmp_instr_id);
13021                 break;
13022
13023         case INSTR_JMP_LT_HI:
13024                 fprintf(f,
13025                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
13026                         "pipeline_instructions[%u].jmp.b_val)",
13027                         jmp_instr_id,
13028                         jmp_instr_id);
13029                 break;
13030
13031         case INSTR_JMP_GT:
13032                 fprintf(f,
13033                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13034                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
13035                         jmp_instr_id,
13036                         jmp_instr_id);
13037                 break;
13038
13039         case INSTR_JMP_GT_MH:
13040                 fprintf(f,
13041                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13042                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13043                         jmp_instr_id,
13044                         jmp_instr_id);
13045                 break;
13046
13047         case INSTR_JMP_GT_HM:
13048                 fprintf(f,
13049                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13050                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
13051                         jmp_instr_id,
13052                         jmp_instr_id);
13053                 break;
13054
13055         case INSTR_JMP_GT_HH:
13056                 fprintf(f,
13057                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13058                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13059                         jmp_instr_id,
13060                         jmp_instr_id);
13061                 break;
13062
13063         case INSTR_JMP_GT_MI:
13064                 fprintf(f,
13065                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13066                         "pipeline_instructions[%u].jmp.b_val)",
13067                         jmp_instr_id,
13068                         jmp_instr_id);
13069                 break;
13070
13071         case INSTR_JMP_GT_HI:
13072                 fprintf(f,
13073                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13074                         "pipeline_instructions[%u].jmp.b_val)",
13075                         jmp_instr_id,
13076                         jmp_instr_id);
13077                 break;
13078
13079         default:
13080                 break;
13081         }
13082
13083         /* Find the instruction group of the jump instruction. */
13084         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
13085         if (!jmp_g)
13086                 return -EINVAL;
13087
13088         /* Find the instruction group of the jump destination instruction. */
13089         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
13090         if (!data)
13091                 return -EINVAL;
13092
13093         instr_id = data - p->instruction_data;
13094
13095         g = instruction_group_list_group_find(igl, instr_id);
13096         if (!g)
13097                 return -EINVAL;
13098
13099         /* Code generation for "near" jump (same instruction group) or "far" jump (different
13100          * instruction group).
13101          */
13102         if (g->group_id == jmp_g->group_id)
13103                 fprintf(f,
13104                         "\n\t\tgoto %s;\n",
13105                         jmp_data->jmp_label);
13106         else
13107                 fprintf(f,
13108                         " {\n"
13109                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
13110                         "\t\treturn;\n"
13111                         "\t}\n\n",
13112                         g->group_id);
13113
13114         return 0;
13115 }
13116
13117 static void
13118 instruction_group_list_codegen(struct instruction_group_list *igl,
13119                                struct rte_swx_pipeline *p,
13120                                FILE *f)
13121 {
13122         struct instruction_group *g;
13123         uint32_t i;
13124         int is_required = 0;
13125
13126         /* Check if code generation is required. */
13127         TAILQ_FOREACH(g, igl, node)
13128                 if (g->first_instr_id < g->last_instr_id)
13129                         is_required = 1;
13130
13131         if (!is_required)
13132                 return;
13133
13134         /* Generate the code for the pipeline instruction array. */
13135         fprintf(f,
13136                 "static const struct instruction pipeline_instructions[] = {\n");
13137
13138         for (i = 0; i < p->n_instructions; i++) {
13139                 struct instruction *instr = &p->instructions[i];
13140                 instruction_export_t func = export_table[instr->type];
13141
13142                 func(instr, f);
13143         }
13144
13145         fprintf(f, "};\n\n");
13146
13147         /* Generate the code for the pipeline functions: one function for each instruction group
13148          * that contains more than one instruction.
13149          */
13150         TAILQ_FOREACH(g, igl, node) {
13151                 struct instruction *last_instr;
13152                 uint32_t j;
13153
13154                 /* Skip if group contains a single instruction. */
13155                 if (g->last_instr_id == g->first_instr_id)
13156                         continue;
13157
13158                 /* Generate new pipeline function. */
13159                 fprintf(f,
13160                         "void\n"
13161                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
13162                         "{\n"
13163                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
13164                         "\n",
13165                         g->group_id);
13166
13167                 /* Generate the code for each pipeline instruction. */
13168                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
13169                         struct instruction *instr = &p->instructions[j];
13170                         struct instruction_data *data = &p->instruction_data[j];
13171
13172                         /* Label, if present. */
13173                         if (data->label[0])
13174                                 fprintf(f, "\n%s : ", data->label);
13175                         else
13176                                 fprintf(f, "\n\t");
13177
13178                         /* TX instruction type. */
13179                         if (instruction_does_tx(instr)) {
13180                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
13181                                 continue;
13182                         }
13183
13184                         /* Jump instruction type. */
13185                         if (instruction_is_jmp(instr)) {
13186                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
13187                                 continue;
13188                         }
13189
13190                         /* Any other instruction type. */
13191                         fprintf(f,
13192                                 "%s(p, t, &pipeline_instructions[%u]);\n",
13193                                 instr_type_to_func(instr),
13194                                 j);
13195                 }
13196
13197                 /* Finalize the generated pipeline function. For some instructions such as TX,
13198                  * emit-many-and-TX and unconditional jump, the next instruction has been already
13199                  * decided unconditionally and the instruction pointer of the current thread set
13200                  * accordingly; for all the other instructions, the instruction pointer must be
13201                  * incremented now.
13202                  */
13203                 last_instr = &p->instructions[g->last_instr_id];
13204
13205                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
13206                         fprintf(f,
13207                                 "thread_ip_inc(p);\n");
13208
13209                 fprintf(f,
13210                         "}\n"
13211                         "\n");
13212         }
13213 }
13214
13215 static uint32_t
13216 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
13217 {
13218         struct instruction_group *g;
13219         uint32_t n_custom_instr = 0;
13220
13221         /* Groups with a single instruction: no function is generated for this group, the group
13222          * keeps its current instruction. Groups with more than two instructions: one function and
13223          * the associated custom instruction get generated for each such group.
13224          */
13225         TAILQ_FOREACH(g, igl, node) {
13226                 if (g->first_instr_id == g->last_instr_id)
13227                         continue;
13228
13229                 n_custom_instr++;
13230         }
13231
13232         return n_custom_instr;
13233 }
13234
13235 static int
13236 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13237 {
13238         struct action *a;
13239         FILE *f = NULL;
13240
13241         /* Create the .c file. */
13242         f = fopen("/tmp/pipeline.c", "w");
13243         if (!f)
13244                 return -EIO;
13245
13246         /* Include the .h file. */
13247         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
13248
13249         /* Add the code for each action. */
13250         TAILQ_FOREACH(a, &p->actions, node) {
13251                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
13252
13253                 action_data_codegen(a, f);
13254
13255                 fprintf(f, "\n");
13256
13257                 action_instr_codegen(a, f);
13258
13259                 fprintf(f, "\n");
13260         }
13261
13262         /* Add the pipeline code. */
13263         instruction_group_list_codegen(igl, p, f);
13264
13265         /* Close the .c file. */
13266         fclose(f);
13267
13268         return 0;
13269 }
13270
13271 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
13272 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
13273 #endif
13274
13275 static int
13276 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13277 {
13278         struct action *a;
13279         struct instruction_group *g;
13280         char *dir_in, *buffer = NULL;
13281         const char *dir_out;
13282         int status = 0;
13283
13284         /* Get the environment variables. */
13285         dir_in = getenv("RTE_INSTALL_DIR");
13286         if (!dir_in) {
13287                 status = -EINVAL;
13288                 goto free;
13289         }
13290
13291         dir_out = "/tmp";
13292
13293         /* Memory allocation for the command buffer. */
13294         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
13295         if (!buffer) {
13296                 status = -ENOMEM;
13297                 goto free;
13298         }
13299
13300         snprintf(buffer,
13301                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
13302                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
13303                  "-I %s/lib/pipeline "
13304                  "-I %s/lib/eal/include "
13305                  "-I %s/lib/eal/x86/include "
13306                  "-I %s/lib/eal/include/generic "
13307                  "-I %s/lib/meter "
13308                  "-I %s/lib/port "
13309                  "-I %s/lib/table "
13310                  "-I %s/lib/pipeline "
13311                  "-I %s/config "
13312                  "-I %s/build "
13313                  "-I %s/lib/eal/linux/include "
13314                  ">%s/pipeline.log 2>&1 "
13315                  "&& "
13316                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
13317                  ">>%s/pipeline.log 2>&1",
13318                  dir_out,
13319                  dir_out,
13320                  dir_in,
13321                  dir_in,
13322                  dir_in,
13323                  dir_in,
13324                  dir_in,
13325                  dir_in,
13326                  dir_in,
13327                  dir_in,
13328                  dir_in,
13329                  dir_in,
13330                  dir_in,
13331                  dir_out,
13332                  dir_out,
13333                  dir_out,
13334                  dir_out);
13335
13336         /* Build the shared object library. */
13337         status = system(buffer);
13338         if (status)
13339                 goto free;
13340
13341         /* Open library. */
13342         snprintf(buffer,
13343                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
13344                  "%s/libpipeline.so",
13345                  dir_out);
13346
13347         p->lib = dlopen(buffer, RTLD_LAZY);
13348         if (!p->lib) {
13349                 status = -EIO;
13350                 goto free;
13351         }
13352
13353         /* Get the action function symbols. */
13354         TAILQ_FOREACH(a, &p->actions, node) {
13355                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
13356
13357                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
13358                 if (!p->action_funcs[a->id]) {
13359                         status = -EINVAL;
13360                         goto free;
13361                 }
13362         }
13363
13364         /* Get the pipeline function symbols. */
13365         TAILQ_FOREACH(g, igl, node) {
13366                 if (g->first_instr_id == g->last_instr_id)
13367                         continue;
13368
13369                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
13370
13371                 g->func = dlsym(p->lib, buffer);
13372                 if (!g->func) {
13373                         status = -EINVAL;
13374                         goto free;
13375                 }
13376         }
13377
13378 free:
13379         if (status && p->lib) {
13380                 dlclose(p->lib);
13381                 p->lib = NULL;
13382         }
13383
13384         free(buffer);
13385
13386         return status;
13387 }
13388
13389 static int
13390 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
13391                       struct instruction_group_list *igl)
13392 {
13393         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
13394
13395         /* Check that enough space is available within the pipeline instruction table to store all
13396          * the custom instructions.
13397          */
13398         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
13399                 return -ENOSPC;
13400
13401         return 0;
13402 }
13403
13404 static void
13405 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13406 {
13407         struct instruction_group *g;
13408         uint32_t i;
13409
13410         /* Pipeline table instructions. */
13411         for (i = 0; i < p->n_instructions; i++) {
13412                 struct instruction *instr = &p->instructions[i];
13413
13414                 if (instr->type == INSTR_TABLE)
13415                         instr->type = INSTR_TABLE_AF;
13416
13417                 if (instr->type == INSTR_LEARNER)
13418                         instr->type = INSTR_LEARNER_AF;
13419         }
13420
13421         /* Pipeline custom instructions. */
13422         i = 0;
13423         TAILQ_FOREACH(g, igl, node) {
13424                 struct instruction *instr = &p->instructions[g->first_instr_id];
13425                 uint32_t j;
13426
13427                 if (g->first_instr_id == g->last_instr_id)
13428                         continue;
13429
13430                 /* Install a new custom instruction. */
13431                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
13432
13433                 /* First instruction of the group: change its type to the new custom instruction. */
13434                 instr->type = INSTR_CUSTOM_0 + i;
13435
13436                 /* All the subsequent instructions of the group: invalidate. */
13437                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
13438                         struct instruction_data *data = &p->instruction_data[j];
13439
13440                         data->invalid = 1;
13441                 }
13442
13443                 i++;
13444         }
13445
13446         /* Remove the invalidated instructions. */
13447         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
13448
13449         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
13450          * instructions that are the only instruction within their group, so they were left
13451          * unmodified).
13452          */
13453         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
13454 }
13455
13456 static int
13457 pipeline_compile(struct rte_swx_pipeline *p)
13458 {
13459         struct instruction_group_list *igl = NULL;
13460         int status = 0;
13461
13462         igl = instruction_group_list_create(p);
13463         if (!igl) {
13464                 status = -ENOMEM;
13465                 goto free;
13466         }
13467
13468         /* Code generation. */
13469         status = pipeline_codegen(p, igl);
13470         if (status)
13471                 goto free;
13472
13473         /* Build and load the shared object library. */
13474         status = pipeline_libload(p, igl);
13475         if (status)
13476                 goto free;
13477
13478         /* Adjust instructions. */
13479         status = pipeline_adjust_check(p, igl);
13480         if (status)
13481                 goto free;
13482
13483         pipeline_adjust(p, igl);
13484
13485 free:
13486         instruction_group_list_free(igl);
13487
13488         return status;
13489 }