lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <dlfcn.h>
   8
   9 #include <rte_jhash.h>
  10 #include <rte_hash_crc.h>
  11
  12 #include <rte_swx_port_ethdev.h>
  13 #include <rte_swx_port_fd.h>
  14 #include <rte_swx_port_ring.h>
  15 #include "rte_swx_port_source_sink.h"
  16
  17 #include <rte_swx_table_em.h>
  18 #include <rte_swx_table_wm.h>
  19
  20 #include "rte_swx_pipeline_internal.h"
  21
  22 #define CHECK(condition, err_code)                                             \
  23 do {                                                                           \
  24         if (!(condition))                                                      \
  25                 return -(err_code);                                            \
  26 } while (0)
  27
  28 #define CHECK_NAME(name, err_code)                                             \
  29         CHECK((name) &&                                                        \
  30               (name)[0] &&                                                     \
  31               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  32               err_code)
  33
  34 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  35         CHECK((instr) &&                                                       \
  36               (instr)[0] &&                                                    \
  37               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  38                RTE_SWX_INSTRUCTION_SIZE),                                      \
  39               err_code)
  40
  41 /*
  42  * Environment.
  43  */
  44 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  45
  46 #include <rte_malloc.h>
  47
  48 static void *
  49 env_malloc(size_t size, size_t alignment, int numa_node)
  50 {
  51         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  52 }
  53
  54 static void
  55 env_free(void *start, size_t size __rte_unused)
  56 {
  57         rte_free(start);
  58 }
  59
  60 #else
  61
  62 #include <numa.h>
  63
  64 static void *
  65 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  66 {
  67         void *start;
  68
  69         if (numa_available() == -1)
  70                 return NULL;
  71
  72         start = numa_alloc_onnode(size, numa_node);
  73         if (!start)
  74                 return NULL;
  75
  76         memset(start, 0, size);
  77         return start;
  78 }
  79
  80 static void
  81 env_free(void *start, size_t size)
  82 {
  83         if (numa_available() == -1)
  84                 return;
  85
  86         numa_free(start, size);
  87 }
  88
  89 #endif
  90
  91 /*
  92  * Struct.
  93  */
  94 static struct struct_type *
  95 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  96 {
  97         struct struct_type *elem;
  98
  99         TAILQ_FOREACH(elem, &p->struct_types, node)
 100                 if (strcmp(elem->name, name) == 0)
 101                         return elem;
 102
 103         return NULL;
 104 }
 105
 106 static struct field *
 107 struct_type_field_find(struct struct_type *st, const char *name)
 108 {
 109         uint32_t i;
 110
 111         for (i = 0; i < st->n_fields; i++) {
 112                 struct field *f = &st->fields[i];
 113
 114                 if (strcmp(f->name, name) == 0)
 115                         return f;
 116         }
 117
 118         return NULL;
 119 }
 120
 121 int
 122 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 123                                       const char *name,
 124                                       struct rte_swx_field_params *fields,
 125                                       uint32_t n_fields,
 126                                       int last_field_has_variable_size)
 127 {
 128         struct struct_type *st;
 129         uint32_t i;
 130
 131         CHECK(p, EINVAL);
 132         CHECK_NAME(name, EINVAL);
 133         CHECK(fields, EINVAL);
 134         CHECK(n_fields, EINVAL);
 135
 136         for (i = 0; i < n_fields; i++) {
 137                 struct rte_swx_field_params *f = &fields[i];
 138                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 139                 uint32_t j;
 140
 141                 CHECK_NAME(f->name, EINVAL);
 142                 CHECK(f->n_bits, EINVAL);
 143                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 144                 CHECK((f->n_bits & 7) == 0, EINVAL);
 145
 146                 for (j = 0; j < i; j++) {
 147                         struct rte_swx_field_params *f_prev = &fields[j];
 148
 149                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 150                 }
 151         }
 152
 153         CHECK(!struct_type_find(p, name), EEXIST);
 154
 155         /* Node allocation. */
 156         st = calloc(1, sizeof(struct struct_type));
 157         CHECK(st, ENOMEM);
 158
 159         st->fields = calloc(n_fields, sizeof(struct field));
 160         if (!st->fields) {
 161                 free(st);
 162                 CHECK(0, ENOMEM);
 163         }
 164
 165         /* Node initialization. */
 166         strcpy(st->name, name);
 167         for (i = 0; i < n_fields; i++) {
 168                 struct field *dst = &st->fields[i];
 169                 struct rte_swx_field_params *src = &fields[i];
 170                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 171
 172                 strcpy(dst->name, src->name);
 173                 dst->n_bits = src->n_bits;
 174                 dst->offset = st->n_bits;
 175                 dst->var_size = var_size;
 176
 177                 st->n_bits += src->n_bits;
 178                 st->n_bits_min += var_size ? 0 : src->n_bits;
 179         }
 180         st->n_fields = n_fields;
 181         st->var_size = last_field_has_variable_size;
 182
 183         /* Node add to tailq. */
 184         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 185
 186         return 0;
 187 }
 188
 189 static int
 190 struct_build(struct rte_swx_pipeline *p)
 191 {
 192         uint32_t i;
 193
 194         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 195                 struct thread *t = &p->threads[i];
 196
 197                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 198                 CHECK(t->structs, ENOMEM);
 199         }
 200
 201         return 0;
 202 }
 203
 204 static void
 205 struct_build_free(struct rte_swx_pipeline *p)
 206 {
 207         uint32_t i;
 208
 209         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 210                 struct thread *t = &p->threads[i];
 211
 212                 free(t->structs);
 213                 t->structs = NULL;
 214         }
 215 }
 216
 217 static void
 218 struct_free(struct rte_swx_pipeline *p)
 219 {
 220         struct_build_free(p);
 221
 222         /* Struct types. */
 223         for ( ; ; ) {
 224                 struct struct_type *elem;
 225
 226                 elem = TAILQ_FIRST(&p->struct_types);
 227                 if (!elem)
 228                         break;
 229
 230                 TAILQ_REMOVE(&p->struct_types, elem, node);
 231                 free(elem->fields);
 232                 free(elem);
 233         }
 234 }
 235
 236 /*
 237  * Input port.
 238  */
 239 static struct port_in_type *
 240 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 241 {
 242         struct port_in_type *elem;
 243
 244         if (!name)
 245                 return NULL;
 246
 247         TAILQ_FOREACH(elem, &p->port_in_types, node)
 248                 if (strcmp(elem->name, name) == 0)
 249                         return elem;
 250
 251         return NULL;
 252 }
 253
 254 int
 255 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 256                                        const char *name,
 257                                        struct rte_swx_port_in_ops *ops)
 258 {
 259         struct port_in_type *elem;
 260
 261         CHECK(p, EINVAL);
 262         CHECK_NAME(name, EINVAL);
 263         CHECK(ops, EINVAL);
 264         CHECK(ops->create, EINVAL);
 265         CHECK(ops->free, EINVAL);
 266         CHECK(ops->pkt_rx, EINVAL);
 267         CHECK(ops->stats_read, EINVAL);
 268
 269         CHECK(!port_in_type_find(p, name), EEXIST);
 270
 271         /* Node allocation. */
 272         elem = calloc(1, sizeof(struct port_in_type));
 273         CHECK(elem, ENOMEM);
 274
 275         /* Node initialization. */
 276         strcpy(elem->name, name);
 277         memcpy(&elem->ops, ops, sizeof(*ops));
 278
 279         /* Node add to tailq. */
 280         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 281
 282         return 0;
 283 }
 284
 285 static struct port_in *
 286 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 287 {
 288         struct port_in *port;
 289
 290         TAILQ_FOREACH(port, &p->ports_in, node)
 291                 if (port->id == port_id)
 292                         return port;
 293
 294         return NULL;
 295 }
 296
 297 int
 298 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 299                                 uint32_t port_id,
 300                                 const char *port_type_name,
 301                                 void *args)
 302 {
 303         struct port_in_type *type = NULL;
 304         struct port_in *port = NULL;
 305         void *obj = NULL;
 306
 307         CHECK(p, EINVAL);
 308
 309         CHECK(!port_in_find(p, port_id), EINVAL);
 310
 311         CHECK_NAME(port_type_name, EINVAL);
 312         type = port_in_type_find(p, port_type_name);
 313         CHECK(type, EINVAL);
 314
 315         obj = type->ops.create(args);
 316         CHECK(obj, ENODEV);
 317
 318         /* Node allocation. */
 319         port = calloc(1, sizeof(struct port_in));
 320         CHECK(port, ENOMEM);
 321
 322         /* Node initialization. */
 323         port->type = type;
 324         port->obj = obj;
 325         port->id = port_id;
 326
 327         /* Node add to tailq. */
 328         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 329         if (p->n_ports_in < port_id + 1)
 330                 p->n_ports_in = port_id + 1;
 331
 332         return 0;
 333 }
 334
 335 static int
 336 port_in_build(struct rte_swx_pipeline *p)
 337 {
 338         struct port_in *port;
 339         uint32_t i;
 340
 341         CHECK(p->n_ports_in, EINVAL);
 342         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 343
 344         for (i = 0; i < p->n_ports_in; i++)
 345                 CHECK(port_in_find(p, i), EINVAL);
 346
 347         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 348         CHECK(p->in, ENOMEM);
 349
 350         TAILQ_FOREACH(port, &p->ports_in, node) {
 351                 struct port_in_runtime *in = &p->in[port->id];
 352
 353                 in->pkt_rx = port->type->ops.pkt_rx;
 354                 in->obj = port->obj;
 355         }
 356
 357         return 0;
 358 }
 359
 360 static void
 361 port_in_build_free(struct rte_swx_pipeline *p)
 362 {
 363         free(p->in);
 364         p->in = NULL;
 365 }
 366
 367 static void
 368 port_in_free(struct rte_swx_pipeline *p)
 369 {
 370         port_in_build_free(p);
 371
 372         /* Input ports. */
 373         for ( ; ; ) {
 374                 struct port_in *port;
 375
 376                 port = TAILQ_FIRST(&p->ports_in);
 377                 if (!port)
 378                         break;
 379
 380                 TAILQ_REMOVE(&p->ports_in, port, node);
 381                 port->type->ops.free(port->obj);
 382                 free(port);
 383         }
 384
 385         /* Input port types. */
 386         for ( ; ; ) {
 387                 struct port_in_type *elem;
 388
 389                 elem = TAILQ_FIRST(&p->port_in_types);
 390                 if (!elem)
 391                         break;
 392
 393                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 394                 free(elem);
 395         }
 396 }
 397
 398 /*
 399  * Output port.
 400  */
 401 static struct port_out_type *
 402 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 403 {
 404         struct port_out_type *elem;
 405
 406         if (!name)
 407                 return NULL;
 408
 409         TAILQ_FOREACH(elem, &p->port_out_types, node)
 410                 if (!strcmp(elem->name, name))
 411                         return elem;
 412
 413         return NULL;
 414 }
 415
 416 int
 417 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 418                                         const char *name,
 419                                         struct rte_swx_port_out_ops *ops)
 420 {
 421         struct port_out_type *elem;
 422
 423         CHECK(p, EINVAL);
 424         CHECK_NAME(name, EINVAL);
 425         CHECK(ops, EINVAL);
 426         CHECK(ops->create, EINVAL);
 427         CHECK(ops->free, EINVAL);
 428         CHECK(ops->pkt_tx, EINVAL);
 429         CHECK(ops->pkt_fast_clone_tx, EINVAL);
 430         CHECK(ops->pkt_clone_tx, EINVAL);
 431         CHECK(ops->stats_read, EINVAL);
 432
 433         CHECK(!port_out_type_find(p, name), EEXIST);
 434
 435         /* Node allocation. */
 436         elem = calloc(1, sizeof(struct port_out_type));
 437         CHECK(elem, ENOMEM);
 438
 439         /* Node initialization. */
 440         strcpy(elem->name, name);
 441         memcpy(&elem->ops, ops, sizeof(*ops));
 442
 443         /* Node add to tailq. */
 444         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 445
 446         return 0;
 447 }
 448
 449 static struct port_out *
 450 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 451 {
 452         struct port_out *port;
 453
 454         TAILQ_FOREACH(port, &p->ports_out, node)
 455                 if (port->id == port_id)
 456                         return port;
 457
 458         return NULL;
 459 }
 460
 461 int
 462 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 463                                  uint32_t port_id,
 464                                  const char *port_type_name,
 465                                  void *args)
 466 {
 467         struct port_out_type *type = NULL;
 468         struct port_out *port = NULL;
 469         void *obj = NULL;
 470
 471         CHECK(p, EINVAL);
 472
 473         CHECK(!port_out_find(p, port_id), EINVAL);
 474
 475         CHECK_NAME(port_type_name, EINVAL);
 476         type = port_out_type_find(p, port_type_name);
 477         CHECK(type, EINVAL);
 478
 479         obj = type->ops.create(args);
 480         CHECK(obj, ENODEV);
 481
 482         /* Node allocation. */
 483         port = calloc(1, sizeof(struct port_out));
 484         CHECK(port, ENOMEM);
 485
 486         /* Node initialization. */
 487         port->type = type;
 488         port->obj = obj;
 489         port->id = port_id;
 490
 491         /* Node add to tailq. */
 492         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 493         if (p->n_ports_out < port_id + 1)
 494                 p->n_ports_out = port_id + 1;
 495
 496         return 0;
 497 }
 498
 499 static int
 500 port_out_build(struct rte_swx_pipeline *p)
 501 {
 502         struct port_out *port;
 503         uint32_t i;
 504
 505         CHECK(p->n_ports_out, EINVAL);
 506
 507         for (i = 0; i < p->n_ports_out; i++)
 508                 CHECK(port_out_find(p, i), EINVAL);
 509
 510         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 511         CHECK(p->out, ENOMEM);
 512
 513         TAILQ_FOREACH(port, &p->ports_out, node) {
 514                 struct port_out_runtime *out = &p->out[port->id];
 515
 516                 out->pkt_tx = port->type->ops.pkt_tx;
 517                 out->pkt_fast_clone_tx = port->type->ops.pkt_fast_clone_tx;
 518                 out->pkt_clone_tx = port->type->ops.pkt_clone_tx;
 519                 out->flush = port->type->ops.flush;
 520                 out->obj = port->obj;
 521         }
 522
 523         return 0;
 524 }
 525
 526 static void
 527 port_out_build_free(struct rte_swx_pipeline *p)
 528 {
 529         free(p->out);
 530         p->out = NULL;
 531 }
 532
 533 static void
 534 port_out_free(struct rte_swx_pipeline *p)
 535 {
 536         port_out_build_free(p);
 537
 538         /* Output ports. */
 539         for ( ; ; ) {
 540                 struct port_out *port;
 541
 542                 port = TAILQ_FIRST(&p->ports_out);
 543                 if (!port)
 544                         break;
 545
 546                 TAILQ_REMOVE(&p->ports_out, port, node);
 547                 port->type->ops.free(port->obj);
 548                 free(port);
 549         }
 550
 551         /* Output port types. */
 552         for ( ; ; ) {
 553                 struct port_out_type *elem;
 554
 555                 elem = TAILQ_FIRST(&p->port_out_types);
 556                 if (!elem)
 557                         break;
 558
 559                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 560                 free(elem);
 561         }
 562 }
 563
 564 /*
 565  * Packet mirroring.
 566  */
 567 int
 568 rte_swx_pipeline_mirroring_config(struct rte_swx_pipeline *p,
 569                                   struct rte_swx_pipeline_mirroring_params *params)
 570 {
 571         CHECK(p, EINVAL);
 572         CHECK(params, EINVAL);
 573         CHECK(params->n_slots, EINVAL);
 574         CHECK(params->n_sessions, EINVAL);
 575         CHECK(!p->build_done, EEXIST);
 576
 577         p->n_mirroring_slots = rte_align32pow2(params->n_slots);
 578         if (p->n_mirroring_slots > 64)
 579                 p->n_mirroring_slots = 64;
 580
 581         p->n_mirroring_sessions = rte_align32pow2(params->n_sessions);
 582
 583         return 0;
 584 }
 585
 586 static void
 587 mirroring_build_free(struct rte_swx_pipeline *p)
 588 {
 589         uint32_t i;
 590
 591         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 592                 struct thread *t = &p->threads[i];
 593
 594                 /* mirroring_slots. */
 595                 free(t->mirroring_slots);
 596                 t->mirroring_slots = NULL;
 597         }
 598
 599         /* mirroring_sessions. */
 600         free(p->mirroring_sessions);
 601         p->mirroring_sessions = NULL;
 602 }
 603
 604 static void
 605 mirroring_free(struct rte_swx_pipeline *p)
 606 {
 607         mirroring_build_free(p);
 608 }
 609
 610 static int
 611 mirroring_build(struct rte_swx_pipeline *p)
 612 {
 613         uint32_t i;
 614
 615         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 616                 struct thread *t = &p->threads[i];
 617
 618                 /* mirroring_slots. */
 619                 t->mirroring_slots = calloc(p->n_mirroring_slots, sizeof(uint32_t));
 620                 if (!t->mirroring_slots)
 621                         goto error;
 622         }
 623
 624         /* mirroring_sessions. */
 625         p->mirroring_sessions = calloc(p->n_mirroring_sessions, sizeof(struct mirroring_session));
 626         if (!p->mirroring_sessions)
 627                 goto error;
 628
 629         return 0;
 630
 631 error:
 632         mirroring_build_free(p);
 633         return -ENOMEM;
 634 }
 635
 636 /*
 637  * Extern object.
 638  */
 639 static struct extern_type *
 640 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 641 {
 642         struct extern_type *elem;
 643
 644         TAILQ_FOREACH(elem, &p->extern_types, node)
 645                 if (strcmp(elem->name, name) == 0)
 646                         return elem;
 647
 648         return NULL;
 649 }
 650
 651 static struct extern_type_member_func *
 652 extern_type_member_func_find(struct extern_type *type, const char *name)
 653 {
 654         struct extern_type_member_func *elem;
 655
 656         TAILQ_FOREACH(elem, &type->funcs, node)
 657                 if (strcmp(elem->name, name) == 0)
 658                         return elem;
 659
 660         return NULL;
 661 }
 662
 663 static struct extern_obj *
 664 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 665 {
 666         struct extern_obj *elem;
 667
 668         TAILQ_FOREACH(elem, &p->extern_objs, node)
 669                 if (strcmp(elem->name, name) == 0)
 670                         return elem;
 671
 672         return NULL;
 673 }
 674
 675 static struct extern_type_member_func *
 676 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 677                              const char *name,
 678                              struct extern_obj **obj)
 679 {
 680         struct extern_obj *object;
 681         struct extern_type_member_func *func;
 682         char *object_name, *func_name;
 683
 684         if (name[0] != 'e' || name[1] != '.')
 685                 return NULL;
 686
 687         object_name = strdup(&name[2]);
 688         if (!object_name)
 689                 return NULL;
 690
 691         func_name = strchr(object_name, '.');
 692         if (!func_name) {
 693                 free(object_name);
 694                 return NULL;
 695         }
 696
 697         *func_name = 0;
 698         func_name++;
 699
 700         object = extern_obj_find(p, object_name);
 701         if (!object) {
 702                 free(object_name);
 703                 return NULL;
 704         }
 705
 706         func = extern_type_member_func_find(object->type, func_name);
 707         if (!func) {
 708                 free(object_name);
 709                 return NULL;
 710         }
 711
 712         if (obj)
 713                 *obj = object;
 714
 715         free(object_name);
 716         return func;
 717 }
 718
 719 static struct field *
 720 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 721                                const char *name,
 722                                struct extern_obj **object)
 723 {
 724         struct extern_obj *obj;
 725         struct field *f;
 726         char *obj_name, *field_name;
 727
 728         if ((name[0] != 'e') || (name[1] != '.'))
 729                 return NULL;
 730
 731         obj_name = strdup(&name[2]);
 732         if (!obj_name)
 733                 return NULL;
 734
 735         field_name = strchr(obj_name, '.');
 736         if (!field_name) {
 737                 free(obj_name);
 738                 return NULL;
 739         }
 740
 741         *field_name = 0;
 742         field_name++;
 743
 744         obj = extern_obj_find(p, obj_name);
 745         if (!obj) {
 746                 free(obj_name);
 747                 return NULL;
 748         }
 749
 750         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 751         if (!f) {
 752                 free(obj_name);
 753                 return NULL;
 754         }
 755
 756         if (object)
 757                 *object = obj;
 758
 759         free(obj_name);
 760         return f;
 761 }
 762
 763 int
 764 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 765         const char *name,
 766         const char *mailbox_struct_type_name,
 767         rte_swx_extern_type_constructor_t constructor,
 768         rte_swx_extern_type_destructor_t destructor)
 769 {
 770         struct extern_type *elem;
 771         struct struct_type *mailbox_struct_type;
 772
 773         CHECK(p, EINVAL);
 774
 775         CHECK_NAME(name, EINVAL);
 776         CHECK(!extern_type_find(p, name), EEXIST);
 777
 778         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 779         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 780         CHECK(mailbox_struct_type, EINVAL);
 781         CHECK(!mailbox_struct_type->var_size, EINVAL);
 782
 783         CHECK(constructor, EINVAL);
 784         CHECK(destructor, EINVAL);
 785
 786         /* Node allocation. */
 787         elem = calloc(1, sizeof(struct extern_type));
 788         CHECK(elem, ENOMEM);
 789
 790         /* Node initialization. */
 791         strcpy(elem->name, name);
 792         elem->mailbox_struct_type = mailbox_struct_type;
 793         elem->constructor = constructor;
 794         elem->destructor = destructor;
 795         TAILQ_INIT(&elem->funcs);
 796
 797         /* Node add to tailq. */
 798         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 799
 800         return 0;
 801 }
 802
 803 int
 804 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 805         const char *extern_type_name,
 806         const char *name,
 807         rte_swx_extern_type_member_func_t member_func)
 808 {
 809         struct extern_type *type;
 810         struct extern_type_member_func *type_member;
 811
 812         CHECK(p, EINVAL);
 813
 814         CHECK_NAME(extern_type_name, EINVAL);
 815         type = extern_type_find(p, extern_type_name);
 816         CHECK(type, EINVAL);
 817         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 818
 819         CHECK_NAME(name, EINVAL);
 820         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 821
 822         CHECK(member_func, EINVAL);
 823
 824         /* Node allocation. */
 825         type_member = calloc(1, sizeof(struct extern_type_member_func));
 826         CHECK(type_member, ENOMEM);
 827
 828         /* Node initialization. */
 829         strcpy(type_member->name, name);
 830         type_member->func = member_func;
 831         type_member->id = type->n_funcs;
 832
 833         /* Node add to tailq. */
 834         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 835         type->n_funcs++;
 836
 837         return 0;
 838 }
 839
 840 int
 841 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 842                                       const char *extern_type_name,
 843                                       const char *name,
 844                                       const char *args)
 845 {
 846         struct extern_type *type;
 847         struct extern_obj *obj;
 848         void *obj_handle;
 849
 850         CHECK(p, EINVAL);
 851
 852         CHECK_NAME(extern_type_name, EINVAL);
 853         type = extern_type_find(p, extern_type_name);
 854         CHECK(type, EINVAL);
 855
 856         CHECK_NAME(name, EINVAL);
 857         CHECK(!extern_obj_find(p, name), EEXIST);
 858
 859         /* Node allocation. */
 860         obj = calloc(1, sizeof(struct extern_obj));
 861         CHECK(obj, ENOMEM);
 862
 863         /* Object construction. */
 864         obj_handle = type->constructor(args);
 865         if (!obj_handle) {
 866                 free(obj);
 867                 CHECK(0, ENODEV);
 868         }
 869
 870         /* Node initialization. */
 871         strcpy(obj->name, name);
 872         obj->type = type;
 873         obj->obj = obj_handle;
 874         obj->struct_id = p->n_structs;
 875         obj->id = p->n_extern_objs;
 876
 877         /* Node add to tailq. */
 878         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 879         p->n_extern_objs++;
 880         p->n_structs++;
 881
 882         return 0;
 883 }
 884
 885 static int
 886 extern_obj_build(struct rte_swx_pipeline *p)
 887 {
 888         uint32_t i;
 889
 890         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 891                 struct thread *t = &p->threads[i];
 892                 struct extern_obj *obj;
 893
 894                 t->extern_objs = calloc(p->n_extern_objs,
 895                                         sizeof(struct extern_obj_runtime));
 896                 CHECK(t->extern_objs, ENOMEM);
 897
 898                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 899                         struct extern_obj_runtime *r =
 900                                 &t->extern_objs[obj->id];
 901                         struct extern_type_member_func *func;
 902                         uint32_t mailbox_size =
 903                                 obj->type->mailbox_struct_type->n_bits / 8;
 904
 905                         r->obj = obj->obj;
 906
 907                         r->mailbox = calloc(1, mailbox_size);
 908                         CHECK(r->mailbox, ENOMEM);
 909
 910                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 911                                 r->funcs[func->id] = func->func;
 912
 913                         t->structs[obj->struct_id] = r->mailbox;
 914                 }
 915         }
 916
 917         return 0;
 918 }
 919
 920 static void
 921 extern_obj_build_free(struct rte_swx_pipeline *p)
 922 {
 923         uint32_t i;
 924
 925         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 926                 struct thread *t = &p->threads[i];
 927                 uint32_t j;
 928
 929                 if (!t->extern_objs)
 930                         continue;
 931
 932                 for (j = 0; j < p->n_extern_objs; j++) {
 933                         struct extern_obj_runtime *r = &t->extern_objs[j];
 934
 935                         free(r->mailbox);
 936                 }
 937
 938                 free(t->extern_objs);
 939                 t->extern_objs = NULL;
 940         }
 941 }
 942
 943 static void
 944 extern_obj_free(struct rte_swx_pipeline *p)
 945 {
 946         extern_obj_build_free(p);
 947
 948         /* Extern objects. */
 949         for ( ; ; ) {
 950                 struct extern_obj *elem;
 951
 952                 elem = TAILQ_FIRST(&p->extern_objs);
 953                 if (!elem)
 954                         break;
 955
 956                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 957                 if (elem->obj)
 958                         elem->type->destructor(elem->obj);
 959                 free(elem);
 960         }
 961
 962         /* Extern types. */
 963         for ( ; ; ) {
 964                 struct extern_type *elem;
 965
 966                 elem = TAILQ_FIRST(&p->extern_types);
 967                 if (!elem)
 968                         break;
 969
 970                 TAILQ_REMOVE(&p->extern_types, elem, node);
 971
 972                 for ( ; ; ) {
 973                         struct extern_type_member_func *func;
 974
 975                         func = TAILQ_FIRST(&elem->funcs);
 976                         if (!func)
 977                                 break;
 978
 979                         TAILQ_REMOVE(&elem->funcs, func, node);
 980                         free(func);
 981                 }
 982
 983                 free(elem);
 984         }
 985 }
 986
 987 /*
 988  * Extern function.
 989  */
 990 static struct extern_func *
 991 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 992 {
 993         struct extern_func *elem;
 994
 995         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 996                 if (strcmp(elem->name, name) == 0)
 997                         return elem;
 998
 999         return NULL;
1000 }
1001
1002 static struct extern_func *
1003 extern_func_parse(struct rte_swx_pipeline *p,
1004                   const char *name)
1005 {
1006         if (name[0] != 'f' || name[1] != '.')
1007                 return NULL;
1008
1009         return extern_func_find(p, &name[2]);
1010 }
1011
1012 static struct field *
1013 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1014                                 const char *name,
1015                                 struct extern_func **function)
1016 {
1017         struct extern_func *func;
1018         struct field *f;
1019         char *func_name, *field_name;
1020
1021         if ((name[0] != 'f') || (name[1] != '.'))
1022                 return NULL;
1023
1024         func_name = strdup(&name[2]);
1025         if (!func_name)
1026                 return NULL;
1027
1028         field_name = strchr(func_name, '.');
1029         if (!field_name) {
1030                 free(func_name);
1031                 return NULL;
1032         }
1033
1034         *field_name = 0;
1035         field_name++;
1036
1037         func = extern_func_find(p, func_name);
1038         if (!func) {
1039                 free(func_name);
1040                 return NULL;
1041         }
1042
1043         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1044         if (!f) {
1045                 free(func_name);
1046                 return NULL;
1047         }
1048
1049         if (function)
1050                 *function = func;
1051
1052         free(func_name);
1053         return f;
1054 }
1055
1056 int
1057 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1058                                       const char *name,
1059                                       const char *mailbox_struct_type_name,
1060                                       rte_swx_extern_func_t func)
1061 {
1062         struct extern_func *f;
1063         struct struct_type *mailbox_struct_type;
1064
1065         CHECK(p, EINVAL);
1066
1067         CHECK_NAME(name, EINVAL);
1068         CHECK(!extern_func_find(p, name), EEXIST);
1069
1070         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1071         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1072         CHECK(mailbox_struct_type, EINVAL);
1073         CHECK(!mailbox_struct_type->var_size, EINVAL);
1074
1075         CHECK(func, EINVAL);
1076
1077         /* Node allocation. */
1078         f = calloc(1, sizeof(struct extern_func));
1079         CHECK(func, ENOMEM);
1080
1081         /* Node initialization. */
1082         strcpy(f->name, name);
1083         f->mailbox_struct_type = mailbox_struct_type;
1084         f->func = func;
1085         f->struct_id = p->n_structs;
1086         f->id = p->n_extern_funcs;
1087
1088         /* Node add to tailq. */
1089         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1090         p->n_extern_funcs++;
1091         p->n_structs++;
1092
1093         return 0;
1094 }
1095
1096 static int
1097 extern_func_build(struct rte_swx_pipeline *p)
1098 {
1099         uint32_t i;
1100
1101         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1102                 struct thread *t = &p->threads[i];
1103                 struct extern_func *func;
1104
1105                 /* Memory allocation. */
1106                 t->extern_funcs = calloc(p->n_extern_funcs,
1107                                          sizeof(struct extern_func_runtime));
1108                 CHECK(t->extern_funcs, ENOMEM);
1109
1110                 /* Extern function. */
1111                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1112                         struct extern_func_runtime *r =
1113                                 &t->extern_funcs[func->id];
1114                         uint32_t mailbox_size =
1115                                 func->mailbox_struct_type->n_bits / 8;
1116
1117                         r->func = func->func;
1118
1119                         r->mailbox = calloc(1, mailbox_size);
1120                         CHECK(r->mailbox, ENOMEM);
1121
1122                         t->structs[func->struct_id] = r->mailbox;
1123                 }
1124         }
1125
1126         return 0;
1127 }
1128
1129 static void
1130 extern_func_build_free(struct rte_swx_pipeline *p)
1131 {
1132         uint32_t i;
1133
1134         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1135                 struct thread *t = &p->threads[i];
1136                 uint32_t j;
1137
1138                 if (!t->extern_funcs)
1139                         continue;
1140
1141                 for (j = 0; j < p->n_extern_funcs; j++) {
1142                         struct extern_func_runtime *r = &t->extern_funcs[j];
1143
1144                         free(r->mailbox);
1145                 }
1146
1147                 free(t->extern_funcs);
1148                 t->extern_funcs = NULL;
1149         }
1150 }
1151
1152 static void
1153 extern_func_free(struct rte_swx_pipeline *p)
1154 {
1155         extern_func_build_free(p);
1156
1157         for ( ; ; ) {
1158                 struct extern_func *elem;
1159
1160                 elem = TAILQ_FIRST(&p->extern_funcs);
1161                 if (!elem)
1162                         break;
1163
1164                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1165                 free(elem);
1166         }
1167 }
1168
1169 /*
1170  * Hash function.
1171  */
1172 static struct hash_func *
1173 hash_func_find(struct rte_swx_pipeline *p, const char *name)
1174 {
1175         struct hash_func *elem;
1176
1177         TAILQ_FOREACH(elem, &p->hash_funcs, node)
1178                 if (strcmp(elem->name, name) == 0)
1179                         return elem;
1180
1181         return NULL;
1182 }
1183
1184 int
1185 rte_swx_pipeline_hash_func_register(struct rte_swx_pipeline *p,
1186                                     const char *name,
1187                                     rte_swx_hash_func_t func)
1188 {
1189         struct hash_func *f;
1190
1191         CHECK(p, EINVAL);
1192
1193         CHECK_NAME(name, EINVAL);
1194         CHECK(!hash_func_find(p, name), EEXIST);
1195
1196         CHECK(func, EINVAL);
1197
1198         /* Node allocation. */
1199         f = calloc(1, sizeof(struct hash_func));
1200         CHECK(func, ENOMEM);
1201
1202         /* Node initialization. */
1203         strcpy(f->name, name);
1204         f->func = func;
1205         f->id = p->n_hash_funcs;
1206
1207         /* Node add to tailq. */
1208         TAILQ_INSERT_TAIL(&p->hash_funcs, f, node);
1209         p->n_hash_funcs++;
1210
1211         return 0;
1212 }
1213
1214 static int
1215 hash_func_build(struct rte_swx_pipeline *p)
1216 {
1217         struct hash_func *func;
1218
1219         /* Memory allocation. */
1220         p->hash_func_runtime = calloc(p->n_hash_funcs, sizeof(struct hash_func_runtime));
1221         CHECK(p->hash_func_runtime, ENOMEM);
1222
1223         /* Hash function. */
1224         TAILQ_FOREACH(func, &p->hash_funcs, node) {
1225                 struct hash_func_runtime *r = &p->hash_func_runtime[func->id];
1226
1227                 r->func = func->func;
1228         }
1229
1230         return 0;
1231 }
1232
1233 static void
1234 hash_func_build_free(struct rte_swx_pipeline *p)
1235 {
1236         free(p->hash_func_runtime);
1237         p->hash_func_runtime = NULL;
1238 }
1239
1240 static void
1241 hash_func_free(struct rte_swx_pipeline *p)
1242 {
1243         hash_func_build_free(p);
1244
1245         for ( ; ; ) {
1246                 struct hash_func *elem;
1247
1248                 elem = TAILQ_FIRST(&p->hash_funcs);
1249                 if (!elem)
1250                         break;
1251
1252                 TAILQ_REMOVE(&p->hash_funcs, elem, node);
1253                 free(elem);
1254         }
1255 }
1256
1257 /*
1258  * Header.
1259  */
1260 static struct header *
1261 header_find(struct rte_swx_pipeline *p, const char *name)
1262 {
1263         struct header *elem;
1264
1265         TAILQ_FOREACH(elem, &p->headers, node)
1266                 if (strcmp(elem->name, name) == 0)
1267                         return elem;
1268
1269         return NULL;
1270 }
1271
1272 static struct header *
1273 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1274 {
1275         struct header *elem;
1276
1277         TAILQ_FOREACH(elem, &p->headers, node)
1278                 if (elem->struct_id == struct_id)
1279                         return elem;
1280
1281         return NULL;
1282 }
1283
1284 static struct header *
1285 header_parse(struct rte_swx_pipeline *p,
1286              const char *name)
1287 {
1288         if (name[0] != 'h' || name[1] != '.')
1289                 return NULL;
1290
1291         return header_find(p, &name[2]);
1292 }
1293
1294 static struct field *
1295 header_field_parse(struct rte_swx_pipeline *p,
1296                    const char *name,
1297                    struct header **header)
1298 {
1299         struct header *h;
1300         struct field *f;
1301         char *header_name, *field_name;
1302
1303         if ((name[0] != 'h') || (name[1] != '.'))
1304                 return NULL;
1305
1306         header_name = strdup(&name[2]);
1307         if (!header_name)
1308                 return NULL;
1309
1310         field_name = strchr(header_name, '.');
1311         if (!field_name) {
1312                 free(header_name);
1313                 return NULL;
1314         }
1315
1316         *field_name = 0;
1317         field_name++;
1318
1319         h = header_find(p, header_name);
1320         if (!h) {
1321                 free(header_name);
1322                 return NULL;
1323         }
1324
1325         f = struct_type_field_find(h->st, field_name);
1326         if (!f) {
1327                 free(header_name);
1328                 return NULL;
1329         }
1330
1331         if (header)
1332                 *header = h;
1333
1334         free(header_name);
1335         return f;
1336 }
1337
1338 int
1339 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1340                                         const char *name,
1341                                         const char *struct_type_name)
1342 {
1343         struct struct_type *st;
1344         struct header *h;
1345         size_t n_headers_max;
1346
1347         CHECK(p, EINVAL);
1348         CHECK_NAME(name, EINVAL);
1349         CHECK_NAME(struct_type_name, EINVAL);
1350
1351         CHECK(!header_find(p, name), EEXIST);
1352
1353         st = struct_type_find(p, struct_type_name);
1354         CHECK(st, EINVAL);
1355
1356         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1357         CHECK(p->n_headers < n_headers_max, ENOSPC);
1358
1359         /* Node allocation. */
1360         h = calloc(1, sizeof(struct header));
1361         CHECK(h, ENOMEM);
1362
1363         /* Node initialization. */
1364         strcpy(h->name, name);
1365         h->st = st;
1366         h->struct_id = p->n_structs;
1367         h->id = p->n_headers;
1368
1369         /* Node add to tailq. */
1370         TAILQ_INSERT_TAIL(&p->headers, h, node);
1371         p->n_headers++;
1372         p->n_structs++;
1373
1374         return 0;
1375 }
1376
1377 static int
1378 header_build(struct rte_swx_pipeline *p)
1379 {
1380         struct header *h;
1381         uint32_t n_bytes = 0, i;
1382
1383         TAILQ_FOREACH(h, &p->headers, node) {
1384                 n_bytes += h->st->n_bits / 8;
1385         }
1386
1387         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1388                 struct thread *t = &p->threads[i];
1389                 uint32_t offset = 0;
1390
1391                 t->headers = calloc(p->n_headers,
1392                                     sizeof(struct header_runtime));
1393                 CHECK(t->headers, ENOMEM);
1394
1395                 t->headers_out = calloc(p->n_headers,
1396                                         sizeof(struct header_out_runtime));
1397                 CHECK(t->headers_out, ENOMEM);
1398
1399                 t->header_storage = calloc(1, n_bytes);
1400                 CHECK(t->header_storage, ENOMEM);
1401
1402                 t->header_out_storage = calloc(1, n_bytes);
1403                 CHECK(t->header_out_storage, ENOMEM);
1404
1405                 TAILQ_FOREACH(h, &p->headers, node) {
1406                         uint8_t *header_storage;
1407                         uint32_t n_bytes =  h->st->n_bits / 8;
1408
1409                         header_storage = &t->header_storage[offset];
1410                         offset += n_bytes;
1411
1412                         t->headers[h->id].ptr0 = header_storage;
1413                         t->headers[h->id].n_bytes = n_bytes;
1414
1415                         t->structs[h->struct_id] = header_storage;
1416                 }
1417         }
1418
1419         return 0;
1420 }
1421
1422 static void
1423 header_build_free(struct rte_swx_pipeline *p)
1424 {
1425         uint32_t i;
1426
1427         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1428                 struct thread *t = &p->threads[i];
1429
1430                 free(t->headers_out);
1431                 t->headers_out = NULL;
1432
1433                 free(t->headers);
1434                 t->headers = NULL;
1435
1436                 free(t->header_out_storage);
1437                 t->header_out_storage = NULL;
1438
1439                 free(t->header_storage);
1440                 t->header_storage = NULL;
1441         }
1442 }
1443
1444 static void
1445 header_free(struct rte_swx_pipeline *p)
1446 {
1447         header_build_free(p);
1448
1449         for ( ; ; ) {
1450                 struct header *elem;
1451
1452                 elem = TAILQ_FIRST(&p->headers);
1453                 if (!elem)
1454                         break;
1455
1456                 TAILQ_REMOVE(&p->headers, elem, node);
1457                 free(elem);
1458         }
1459 }
1460
1461 /*
1462  * Meta-data.
1463  */
1464 static struct field *
1465 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1466 {
1467         if (!p->metadata_st)
1468                 return NULL;
1469
1470         if (name[0] != 'm' || name[1] != '.')
1471                 return NULL;
1472
1473         return struct_type_field_find(p->metadata_st, &name[2]);
1474 }
1475
1476 int
1477 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1478                                           const char *struct_type_name)
1479 {
1480         struct struct_type *st = NULL;
1481
1482         CHECK(p, EINVAL);
1483
1484         CHECK_NAME(struct_type_name, EINVAL);
1485         st  = struct_type_find(p, struct_type_name);
1486         CHECK(st, EINVAL);
1487         CHECK(!st->var_size, EINVAL);
1488         CHECK(!p->metadata_st, EINVAL);
1489
1490         p->metadata_st = st;
1491         p->metadata_struct_id = p->n_structs;
1492
1493         p->n_structs++;
1494
1495         return 0;
1496 }
1497
1498 static int
1499 metadata_build(struct rte_swx_pipeline *p)
1500 {
1501         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1502         uint32_t i;
1503
1504         /* Thread-level initialization. */
1505         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1506                 struct thread *t = &p->threads[i];
1507                 uint8_t *metadata;
1508
1509                 metadata = calloc(1, n_bytes);
1510                 CHECK(metadata, ENOMEM);
1511
1512                 t->metadata = metadata;
1513                 t->structs[p->metadata_struct_id] = metadata;
1514         }
1515
1516         return 0;
1517 }
1518
1519 static void
1520 metadata_build_free(struct rte_swx_pipeline *p)
1521 {
1522         uint32_t i;
1523
1524         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1525                 struct thread *t = &p->threads[i];
1526
1527                 free(t->metadata);
1528                 t->metadata = NULL;
1529         }
1530 }
1531
1532 static void
1533 metadata_free(struct rte_swx_pipeline *p)
1534 {
1535         metadata_build_free(p);
1536 }
1537
1538 /*
1539  * Instruction.
1540  */
1541 static int
1542 instruction_is_tx(enum instruction_type type)
1543 {
1544         switch (type) {
1545         case INSTR_TX:
1546         case INSTR_TX_I:
1547         case INSTR_DROP:
1548                 return 1;
1549
1550         default:
1551                 return 0;
1552         }
1553 }
1554
1555 static int
1556 instruction_does_tx(struct instruction *instr)
1557 {
1558         switch (instr->type) {
1559         case INSTR_TX:
1560         case INSTR_TX_I:
1561         case INSTR_DROP:
1562         case INSTR_HDR_EMIT_TX:
1563         case INSTR_HDR_EMIT2_TX:
1564         case INSTR_HDR_EMIT3_TX:
1565         case INSTR_HDR_EMIT4_TX:
1566         case INSTR_HDR_EMIT5_TX:
1567         case INSTR_HDR_EMIT6_TX:
1568         case INSTR_HDR_EMIT7_TX:
1569         case INSTR_HDR_EMIT8_TX:
1570                 return 1;
1571         default:
1572                 return 0;
1573         }
1574 }
1575
1576 static int
1577 instruction_is_jmp(struct instruction *instr)
1578 {
1579         switch (instr->type) {
1580         case INSTR_JMP:
1581         case INSTR_JMP_VALID:
1582         case INSTR_JMP_INVALID:
1583         case INSTR_JMP_HIT:
1584         case INSTR_JMP_MISS:
1585         case INSTR_JMP_ACTION_HIT:
1586         case INSTR_JMP_ACTION_MISS:
1587         case INSTR_JMP_EQ:
1588         case INSTR_JMP_EQ_MH:
1589         case INSTR_JMP_EQ_HM:
1590         case INSTR_JMP_EQ_HH:
1591         case INSTR_JMP_EQ_I:
1592         case INSTR_JMP_NEQ:
1593         case INSTR_JMP_NEQ_MH:
1594         case INSTR_JMP_NEQ_HM:
1595         case INSTR_JMP_NEQ_HH:
1596         case INSTR_JMP_NEQ_I:
1597         case INSTR_JMP_LT:
1598         case INSTR_JMP_LT_MH:
1599         case INSTR_JMP_LT_HM:
1600         case INSTR_JMP_LT_HH:
1601         case INSTR_JMP_LT_MI:
1602         case INSTR_JMP_LT_HI:
1603         case INSTR_JMP_GT:
1604         case INSTR_JMP_GT_MH:
1605         case INSTR_JMP_GT_HM:
1606         case INSTR_JMP_GT_HH:
1607         case INSTR_JMP_GT_MI:
1608         case INSTR_JMP_GT_HI:
1609                 return 1;
1610
1611         default:
1612                 return 0;
1613         }
1614 }
1615
1616 static int
1617 instruction_does_thread_yield(struct instruction *instr)
1618 {
1619         switch (instr->type) {
1620         case INSTR_RX:
1621         case INSTR_TABLE:
1622         case INSTR_TABLE_AF:
1623         case INSTR_SELECTOR:
1624         case INSTR_LEARNER:
1625         case INSTR_LEARNER_AF:
1626         case INSTR_EXTERN_OBJ:
1627         case INSTR_EXTERN_FUNC:
1628                 return 1;
1629         default:
1630                 return 0;
1631         }
1632 }
1633
1634 static struct field *
1635 action_field_parse(struct action *action, const char *name);
1636
1637 static struct field *
1638 struct_field_parse(struct rte_swx_pipeline *p,
1639                    struct action *action,
1640                    const char *name,
1641                    uint32_t *struct_id)
1642 {
1643         struct field *f;
1644
1645         switch (name[0]) {
1646         case 'h':
1647         {
1648                 struct header *header;
1649
1650                 f = header_field_parse(p, name, &header);
1651                 if (!f)
1652                         return NULL;
1653
1654                 *struct_id = header->struct_id;
1655                 return f;
1656         }
1657
1658         case 'm':
1659         {
1660                 f = metadata_field_parse(p, name);
1661                 if (!f)
1662                         return NULL;
1663
1664                 *struct_id = p->metadata_struct_id;
1665                 return f;
1666         }
1667
1668         case 't':
1669         {
1670                 if (!action)
1671                         return NULL;
1672
1673                 f = action_field_parse(action, name);
1674                 if (!f)
1675                         return NULL;
1676
1677                 *struct_id = 0;
1678                 return f;
1679         }
1680
1681         case 'e':
1682         {
1683                 struct extern_obj *obj;
1684
1685                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1686                 if (!f)
1687                         return NULL;
1688
1689                 *struct_id = obj->struct_id;
1690                 return f;
1691         }
1692
1693         case 'f':
1694         {
1695                 struct extern_func *func;
1696
1697                 f = extern_func_mailbox_field_parse(p, name, &func);
1698                 if (!f)
1699                         return NULL;
1700
1701                 *struct_id = func->struct_id;
1702                 return f;
1703         }
1704
1705         default:
1706                 return NULL;
1707         }
1708 }
1709
1710 /*
1711  * rx.
1712  */
1713 static int
1714 instr_rx_translate(struct rte_swx_pipeline *p,
1715                    struct action *action,
1716                    char **tokens,
1717                    int n_tokens,
1718                    struct instruction *instr,
1719                    struct instruction_data *data __rte_unused)
1720 {
1721         struct field *f;
1722
1723         CHECK(!action, EINVAL);
1724         CHECK(n_tokens == 2, EINVAL);
1725
1726         f = metadata_field_parse(p, tokens[1]);
1727         CHECK(f, EINVAL);
1728
1729         instr->type = INSTR_RX;
1730         instr->io.io.offset = f->offset / 8;
1731         instr->io.io.n_bits = f->n_bits;
1732         return 0;
1733 }
1734
1735 /*
1736  * tx.
1737  */
1738 static int
1739 instr_tx_translate(struct rte_swx_pipeline *p,
1740                    struct action *action __rte_unused,
1741                    char **tokens,
1742                    int n_tokens,
1743                    struct instruction *instr,
1744                    struct instruction_data *data __rte_unused)
1745 {
1746         char *port = tokens[1];
1747         struct field *f;
1748         uint32_t port_val;
1749
1750         CHECK(n_tokens == 2, EINVAL);
1751
1752         f = metadata_field_parse(p, port);
1753         if (f) {
1754                 instr->type = INSTR_TX;
1755                 instr->io.io.offset = f->offset / 8;
1756                 instr->io.io.n_bits = f->n_bits;
1757                 return 0;
1758         }
1759
1760         /* TX_I. */
1761         port_val = strtoul(port, &port, 0);
1762         CHECK(!port[0], EINVAL);
1763
1764         instr->type = INSTR_TX_I;
1765         instr->io.io.val = port_val;
1766         return 0;
1767 }
1768
1769 static int
1770 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1771                      struct action *action __rte_unused,
1772                      char **tokens __rte_unused,
1773                      int n_tokens,
1774                      struct instruction *instr,
1775                      struct instruction_data *data __rte_unused)
1776 {
1777         CHECK(n_tokens == 1, EINVAL);
1778
1779         /* DROP. */
1780         instr->type = INSTR_DROP;
1781         return 0;
1782 }
1783
1784 static inline void
1785 instr_tx_exec(struct rte_swx_pipeline *p)
1786 {
1787         struct thread *t = &p->threads[p->thread_id];
1788         struct instruction *ip = t->ip;
1789
1790         __instr_tx_exec(p, t, ip);
1791
1792         /* Thread. */
1793         thread_ip_reset(p, t);
1794         instr_rx_exec(p);
1795 }
1796
1797 static inline void
1798 instr_tx_i_exec(struct rte_swx_pipeline *p)
1799 {
1800         struct thread *t = &p->threads[p->thread_id];
1801         struct instruction *ip = t->ip;
1802
1803         __instr_tx_i_exec(p, t, ip);
1804
1805         /* Thread. */
1806         thread_ip_reset(p, t);
1807         instr_rx_exec(p);
1808 }
1809
1810 static inline void
1811 instr_drop_exec(struct rte_swx_pipeline *p)
1812 {
1813         struct thread *t = &p->threads[p->thread_id];
1814         struct instruction *ip = t->ip;
1815
1816         __instr_drop_exec(p, t, ip);
1817
1818         /* Thread. */
1819         thread_ip_reset(p, t);
1820         instr_rx_exec(p);
1821 }
1822
1823 /*
1824  * mirror.
1825  */
1826 static int
1827 instr_mirror_translate(struct rte_swx_pipeline *p,
1828                        struct action *action,
1829                        char **tokens,
1830                        int n_tokens,
1831                        struct instruction *instr,
1832                        struct instruction_data *data __rte_unused)
1833 {
1834         char *dst = tokens[1], *src = tokens[2];
1835         struct field *fdst, *fsrc;
1836         uint32_t dst_struct_id = 0, src_struct_id = 0;
1837
1838         CHECK(n_tokens == 3, EINVAL);
1839
1840         fdst = struct_field_parse(p, action, dst, &dst_struct_id);
1841         CHECK(fdst, EINVAL);
1842         CHECK(dst[0] != 'h', EINVAL);
1843         CHECK(!fdst->var_size, EINVAL);
1844
1845         fsrc = struct_field_parse(p, action, src, &src_struct_id);
1846         CHECK(fsrc, EINVAL);
1847         CHECK(src[0] != 'h', EINVAL);
1848         CHECK(!fsrc->var_size, EINVAL);
1849
1850         instr->type = INSTR_MIRROR;
1851         instr->mirror.dst.struct_id = (uint8_t)dst_struct_id;
1852         instr->mirror.dst.n_bits = fdst->n_bits;
1853         instr->mirror.dst.offset = fdst->offset / 8;
1854         instr->mirror.src.struct_id = (uint8_t)src_struct_id;
1855         instr->mirror.src.n_bits = fsrc->n_bits;
1856         instr->mirror.src.offset = fsrc->offset / 8;
1857
1858         return 0;
1859 }
1860
1861 static inline void
1862 instr_mirror_exec(struct rte_swx_pipeline *p)
1863 {
1864         struct thread *t = &p->threads[p->thread_id];
1865         struct instruction *ip = t->ip;
1866
1867         __instr_mirror_exec(p, t, ip);
1868
1869         /* Thread. */
1870         thread_ip_inc(p);
1871 }
1872
1873 /*
1874  * recirculate.
1875  */
1876 static int
1877 instr_recirculate_translate(struct rte_swx_pipeline *p __rte_unused,
1878                             struct action *action __rte_unused,
1879                             char **tokens __rte_unused,
1880                             int n_tokens,
1881                             struct instruction *instr,
1882                             struct instruction_data *data __rte_unused)
1883 {
1884         CHECK(n_tokens == 1, EINVAL);
1885
1886         instr->type = INSTR_RECIRCULATE;
1887         return 0;
1888 }
1889
1890 static int
1891 instr_recircid_translate(struct rte_swx_pipeline *p,
1892                          struct action *action __rte_unused,
1893                          char **tokens,
1894                          int n_tokens,
1895                          struct instruction *instr,
1896                          struct instruction_data *data __rte_unused)
1897 {
1898         struct field *f;
1899
1900         CHECK(n_tokens == 2, EINVAL);
1901
1902         f = metadata_field_parse(p, tokens[1]);
1903         CHECK(f, EINVAL);
1904
1905         instr->type = INSTR_RECIRCID;
1906         instr->io.io.offset = f->offset / 8;
1907         instr->io.io.n_bits = f->n_bits;
1908         return 0;
1909 }
1910
1911 static inline void
1912 instr_recirculate_exec(struct rte_swx_pipeline *p)
1913 {
1914         struct thread *t = &p->threads[p->thread_id];
1915         struct instruction *ip = t->ip;
1916
1917         __instr_recirculate_exec(p, t, ip);
1918
1919         /* Thread. */
1920         thread_ip_inc(p);
1921 }
1922
1923 static inline void
1924 instr_recircid_exec(struct rte_swx_pipeline *p)
1925 {
1926         struct thread *t = &p->threads[p->thread_id];
1927         struct instruction *ip = t->ip;
1928
1929         __instr_recircid_exec(p, t, ip);
1930
1931         /* Thread. */
1932         thread_ip_inc(p);
1933 }
1934
1935 /*
1936  * extract.
1937  */
1938 static int
1939 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1940                             struct action *action,
1941                             char **tokens,
1942                             int n_tokens,
1943                             struct instruction *instr,
1944                             struct instruction_data *data __rte_unused)
1945 {
1946         struct header *h;
1947
1948         CHECK(!action, EINVAL);
1949         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1950
1951         h = header_parse(p, tokens[1]);
1952         CHECK(h, EINVAL);
1953
1954         if (n_tokens == 2) {
1955                 CHECK(!h->st->var_size, EINVAL);
1956
1957                 instr->type = INSTR_HDR_EXTRACT;
1958                 instr->io.hdr.header_id[0] = h->id;
1959                 instr->io.hdr.struct_id[0] = h->struct_id;
1960                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1961         } else {
1962                 struct field *mf;
1963
1964                 CHECK(h->st->var_size, EINVAL);
1965
1966                 mf = metadata_field_parse(p, tokens[2]);
1967                 CHECK(mf, EINVAL);
1968                 CHECK(!mf->var_size, EINVAL);
1969
1970                 instr->type = INSTR_HDR_EXTRACT_M;
1971                 instr->io.io.offset = mf->offset / 8;
1972                 instr->io.io.n_bits = mf->n_bits;
1973                 instr->io.hdr.header_id[0] = h->id;
1974                 instr->io.hdr.struct_id[0] = h->struct_id;
1975                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1976         }
1977
1978         return 0;
1979 }
1980
1981 static int
1982 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1983                               struct action *action,
1984                               char **tokens,
1985                               int n_tokens,
1986                               struct instruction *instr,
1987                               struct instruction_data *data __rte_unused)
1988 {
1989         struct header *h;
1990
1991         CHECK(!action, EINVAL);
1992         CHECK(n_tokens == 2, EINVAL);
1993
1994         h = header_parse(p, tokens[1]);
1995         CHECK(h, EINVAL);
1996         CHECK(!h->st->var_size, EINVAL);
1997
1998         instr->type = INSTR_HDR_LOOKAHEAD;
1999         instr->io.hdr.header_id[0] = h->id;
2000         instr->io.hdr.struct_id[0] = h->struct_id;
2001         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
2002
2003         return 0;
2004 }
2005
2006 static inline void
2007 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2008 {
2009         struct thread *t = &p->threads[p->thread_id];
2010         struct instruction *ip = t->ip;
2011
2012         __instr_hdr_extract_exec(p, t, ip);
2013
2014         /* Thread. */
2015         thread_ip_inc(p);
2016 }
2017
2018 static inline void
2019 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2020 {
2021         struct thread *t = &p->threads[p->thread_id];
2022         struct instruction *ip = t->ip;
2023
2024         __instr_hdr_extract2_exec(p, t, ip);
2025
2026         /* Thread. */
2027         thread_ip_inc(p);
2028 }
2029
2030 static inline void
2031 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2032 {
2033         struct thread *t = &p->threads[p->thread_id];
2034         struct instruction *ip = t->ip;
2035
2036         __instr_hdr_extract3_exec(p, t, ip);
2037
2038         /* Thread. */
2039         thread_ip_inc(p);
2040 }
2041
2042 static inline void
2043 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2044 {
2045         struct thread *t = &p->threads[p->thread_id];
2046         struct instruction *ip = t->ip;
2047
2048         __instr_hdr_extract4_exec(p, t, ip);
2049
2050         /* Thread. */
2051         thread_ip_inc(p);
2052 }
2053
2054 static inline void
2055 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2056 {
2057         struct thread *t = &p->threads[p->thread_id];
2058         struct instruction *ip = t->ip;
2059
2060         __instr_hdr_extract5_exec(p, t, ip);
2061
2062         /* Thread. */
2063         thread_ip_inc(p);
2064 }
2065
2066 static inline void
2067 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2068 {
2069         struct thread *t = &p->threads[p->thread_id];
2070         struct instruction *ip = t->ip;
2071
2072         __instr_hdr_extract6_exec(p, t, ip);
2073
2074         /* Thread. */
2075         thread_ip_inc(p);
2076 }
2077
2078 static inline void
2079 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2080 {
2081         struct thread *t = &p->threads[p->thread_id];
2082         struct instruction *ip = t->ip;
2083
2084         __instr_hdr_extract7_exec(p, t, ip);
2085
2086         /* Thread. */
2087         thread_ip_inc(p);
2088 }
2089
2090 static inline void
2091 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2092 {
2093         struct thread *t = &p->threads[p->thread_id];
2094         struct instruction *ip = t->ip;
2095
2096         __instr_hdr_extract8_exec(p, t, ip);
2097
2098         /* Thread. */
2099         thread_ip_inc(p);
2100 }
2101
2102 static inline void
2103 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
2104 {
2105         struct thread *t = &p->threads[p->thread_id];
2106         struct instruction *ip = t->ip;
2107
2108         __instr_hdr_extract_m_exec(p, t, ip);
2109
2110         /* Thread. */
2111         thread_ip_inc(p);
2112 }
2113
2114 static inline void
2115 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
2116 {
2117         struct thread *t = &p->threads[p->thread_id];
2118         struct instruction *ip = t->ip;
2119
2120         __instr_hdr_lookahead_exec(p, t, ip);
2121
2122         /* Thread. */
2123         thread_ip_inc(p);
2124 }
2125
2126 /*
2127  * emit.
2128  */
2129 static int
2130 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2131                          struct action *action __rte_unused,
2132                          char **tokens,
2133                          int n_tokens,
2134                          struct instruction *instr,
2135                          struct instruction_data *data __rte_unused)
2136 {
2137         struct header *h;
2138
2139         CHECK(n_tokens == 2, EINVAL);
2140
2141         h = header_parse(p, tokens[1]);
2142         CHECK(h, EINVAL);
2143
2144         instr->type = INSTR_HDR_EMIT;
2145         instr->io.hdr.header_id[0] = h->id;
2146         instr->io.hdr.struct_id[0] = h->struct_id;
2147         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2148         return 0;
2149 }
2150
2151 static inline void
2152 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2153 {
2154         struct thread *t = &p->threads[p->thread_id];
2155         struct instruction *ip = t->ip;
2156
2157         __instr_hdr_emit_exec(p, t, ip);
2158
2159         /* Thread. */
2160         thread_ip_inc(p);
2161 }
2162
2163 static inline void
2164 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2165 {
2166         struct thread *t = &p->threads[p->thread_id];
2167         struct instruction *ip = t->ip;
2168
2169         __instr_hdr_emit_tx_exec(p, t, ip);
2170
2171         /* Thread. */
2172         thread_ip_reset(p, t);
2173         instr_rx_exec(p);
2174 }
2175
2176 static inline void
2177 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2178 {
2179         struct thread *t = &p->threads[p->thread_id];
2180         struct instruction *ip = t->ip;
2181
2182         __instr_hdr_emit2_tx_exec(p, t, ip);
2183
2184         /* Thread. */
2185         thread_ip_reset(p, t);
2186         instr_rx_exec(p);
2187 }
2188
2189 static inline void
2190 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2191 {
2192         struct thread *t = &p->threads[p->thread_id];
2193         struct instruction *ip = t->ip;
2194
2195         __instr_hdr_emit3_tx_exec(p, t, ip);
2196
2197         /* Thread. */
2198         thread_ip_reset(p, t);
2199         instr_rx_exec(p);
2200 }
2201
2202 static inline void
2203 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2204 {
2205         struct thread *t = &p->threads[p->thread_id];
2206         struct instruction *ip = t->ip;
2207
2208         __instr_hdr_emit4_tx_exec(p, t, ip);
2209
2210         /* Thread. */
2211         thread_ip_reset(p, t);
2212         instr_rx_exec(p);
2213 }
2214
2215 static inline void
2216 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2217 {
2218         struct thread *t = &p->threads[p->thread_id];
2219         struct instruction *ip = t->ip;
2220
2221         __instr_hdr_emit5_tx_exec(p, t, ip);
2222
2223         /* Thread. */
2224         thread_ip_reset(p, t);
2225         instr_rx_exec(p);
2226 }
2227
2228 static inline void
2229 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2230 {
2231         struct thread *t = &p->threads[p->thread_id];
2232         struct instruction *ip = t->ip;
2233
2234         __instr_hdr_emit6_tx_exec(p, t, ip);
2235
2236         /* Thread. */
2237         thread_ip_reset(p, t);
2238         instr_rx_exec(p);
2239 }
2240
2241 static inline void
2242 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2243 {
2244         struct thread *t = &p->threads[p->thread_id];
2245         struct instruction *ip = t->ip;
2246
2247         __instr_hdr_emit7_tx_exec(p, t, ip);
2248
2249         /* Thread. */
2250         thread_ip_reset(p, t);
2251         instr_rx_exec(p);
2252 }
2253
2254 static inline void
2255 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2256 {
2257         struct thread *t = &p->threads[p->thread_id];
2258         struct instruction *ip = t->ip;
2259
2260         __instr_hdr_emit8_tx_exec(p, t, ip);
2261
2262         /* Thread. */
2263         thread_ip_reset(p, t);
2264         instr_rx_exec(p);
2265 }
2266
2267 /*
2268  * validate.
2269  */
2270 static int
2271 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2272                              struct action *action __rte_unused,
2273                              char **tokens,
2274                              int n_tokens,
2275                              struct instruction *instr,
2276                              struct instruction_data *data __rte_unused)
2277 {
2278         struct header *h;
2279
2280         CHECK(n_tokens == 2, EINVAL);
2281
2282         h = header_parse(p, tokens[1]);
2283         CHECK(h, EINVAL);
2284
2285         instr->type = INSTR_HDR_VALIDATE;
2286         instr->valid.header_id = h->id;
2287         instr->valid.struct_id = h->struct_id;
2288         return 0;
2289 }
2290
2291 static inline void
2292 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2293 {
2294         struct thread *t = &p->threads[p->thread_id];
2295         struct instruction *ip = t->ip;
2296
2297         __instr_hdr_validate_exec(p, t, ip);
2298
2299         /* Thread. */
2300         thread_ip_inc(p);
2301 }
2302
2303 /*
2304  * invalidate.
2305  */
2306 static int
2307 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2308                                struct action *action __rte_unused,
2309                                char **tokens,
2310                                int n_tokens,
2311                                struct instruction *instr,
2312                                struct instruction_data *data __rte_unused)
2313 {
2314         struct header *h;
2315
2316         CHECK(n_tokens == 2, EINVAL);
2317
2318         h = header_parse(p, tokens[1]);
2319         CHECK(h, EINVAL);
2320
2321         instr->type = INSTR_HDR_INVALIDATE;
2322         instr->valid.header_id = h->id;
2323         return 0;
2324 }
2325
2326 static inline void
2327 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2328 {
2329         struct thread *t = &p->threads[p->thread_id];
2330         struct instruction *ip = t->ip;
2331
2332         __instr_hdr_invalidate_exec(p, t, ip);
2333
2334         /* Thread. */
2335         thread_ip_inc(p);
2336 }
2337
2338 /*
2339  * table.
2340  */
2341 static struct table *
2342 table_find(struct rte_swx_pipeline *p, const char *name);
2343
2344 static struct selector *
2345 selector_find(struct rte_swx_pipeline *p, const char *name);
2346
2347 static struct learner *
2348 learner_find(struct rte_swx_pipeline *p, const char *name);
2349
2350 static int
2351 instr_table_translate(struct rte_swx_pipeline *p,
2352                       struct action *action,
2353                       char **tokens,
2354                       int n_tokens,
2355                       struct instruction *instr,
2356                       struct instruction_data *data __rte_unused)
2357 {
2358         struct table *t;
2359         struct selector *s;
2360         struct learner *l;
2361
2362         CHECK(!action, EINVAL);
2363         CHECK(n_tokens == 2, EINVAL);
2364
2365         t = table_find(p, tokens[1]);
2366         if (t) {
2367                 instr->type = INSTR_TABLE;
2368                 instr->table.table_id = t->id;
2369                 return 0;
2370         }
2371
2372         s = selector_find(p, tokens[1]);
2373         if (s) {
2374                 instr->type = INSTR_SELECTOR;
2375                 instr->table.table_id = s->id;
2376                 return 0;
2377         }
2378
2379         l = learner_find(p, tokens[1]);
2380         if (l) {
2381                 instr->type = INSTR_LEARNER;
2382                 instr->table.table_id = l->id;
2383                 return 0;
2384         }
2385
2386         CHECK(0, EINVAL);
2387 }
2388
2389 static inline void
2390 instr_table_exec(struct rte_swx_pipeline *p)
2391 {
2392         struct thread *t = &p->threads[p->thread_id];
2393         struct instruction *ip = t->ip;
2394         uint32_t table_id = ip->table.table_id;
2395         struct rte_swx_table_state *ts = &t->table_state[table_id];
2396         struct table_runtime *table = &t->tables[table_id];
2397         struct table_statistics *stats = &p->table_stats[table_id];
2398         uint64_t action_id, n_pkts_hit, n_pkts_action;
2399         uint8_t *action_data;
2400         int done, hit;
2401
2402         /* Table. */
2403         done = table->func(ts->obj,
2404                            table->mailbox,
2405                            table->key,
2406                            &action_id,
2407                            &action_data,
2408                            &hit);
2409         if (!done) {
2410                 /* Thread. */
2411                 TRACE("[Thread %2u] table %u (not finalized)\n",
2412                       p->thread_id,
2413                       table_id);
2414
2415                 thread_yield(p);
2416                 return;
2417         }
2418
2419         action_id = hit ? action_id : ts->default_action_id;
2420         action_data = hit ? action_data : ts->default_action_data;
2421         n_pkts_hit = stats->n_pkts_hit[hit];
2422         n_pkts_action = stats->n_pkts_action[action_id];
2423
2424         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2425               p->thread_id,
2426               table_id,
2427               hit ? "hit" : "miss",
2428               (uint32_t)action_id);
2429
2430         t->action_id = action_id;
2431         t->structs[0] = action_data;
2432         t->hit = hit;
2433         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2434         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2435
2436         /* Thread. */
2437         thread_ip_action_call(p, t, action_id);
2438 }
2439
2440 static inline void
2441 instr_table_af_exec(struct rte_swx_pipeline *p)
2442 {
2443         struct thread *t = &p->threads[p->thread_id];
2444         struct instruction *ip = t->ip;
2445         uint32_t table_id = ip->table.table_id;
2446         struct rte_swx_table_state *ts = &t->table_state[table_id];
2447         struct table_runtime *table = &t->tables[table_id];
2448         struct table_statistics *stats = &p->table_stats[table_id];
2449         uint64_t action_id, n_pkts_hit, n_pkts_action;
2450         uint8_t *action_data;
2451         action_func_t action_func;
2452         int done, hit;
2453
2454         /* Table. */
2455         done = table->func(ts->obj,
2456                            table->mailbox,
2457                            table->key,
2458                            &action_id,
2459                            &action_data,
2460                            &hit);
2461         if (!done) {
2462                 /* Thread. */
2463                 TRACE("[Thread %2u] table %u (not finalized)\n",
2464                       p->thread_id,
2465                       table_id);
2466
2467                 thread_yield(p);
2468                 return;
2469         }
2470
2471         action_id = hit ? action_id : ts->default_action_id;
2472         action_data = hit ? action_data : ts->default_action_data;
2473         action_func = p->action_funcs[action_id];
2474         n_pkts_hit = stats->n_pkts_hit[hit];
2475         n_pkts_action = stats->n_pkts_action[action_id];
2476
2477         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2478               p->thread_id,
2479               table_id,
2480               hit ? "hit" : "miss",
2481               (uint32_t)action_id);
2482
2483         t->action_id = action_id;
2484         t->structs[0] = action_data;
2485         t->hit = hit;
2486         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2487         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2488
2489         /* Thread. */
2490         thread_ip_inc(p);
2491
2492         /* Action. */
2493         action_func(p);
2494 }
2495
2496 static inline void
2497 instr_selector_exec(struct rte_swx_pipeline *p)
2498 {
2499         struct thread *t = &p->threads[p->thread_id];
2500         struct instruction *ip = t->ip;
2501         uint32_t selector_id = ip->table.table_id;
2502         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2503         struct selector_runtime *selector = &t->selectors[selector_id];
2504         struct selector_statistics *stats = &p->selector_stats[selector_id];
2505         uint64_t n_pkts = stats->n_pkts;
2506         int done;
2507
2508         /* Table. */
2509         done = rte_swx_table_selector_select(ts->obj,
2510                            selector->mailbox,
2511                            selector->group_id_buffer,
2512                            selector->selector_buffer,
2513                            selector->member_id_buffer);
2514         if (!done) {
2515                 /* Thread. */
2516                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2517                       p->thread_id,
2518                       selector_id);
2519
2520                 thread_yield(p);
2521                 return;
2522         }
2523
2524
2525         TRACE("[Thread %2u] selector %u\n",
2526               p->thread_id,
2527               selector_id);
2528
2529         stats->n_pkts = n_pkts + 1;
2530
2531         /* Thread. */
2532         thread_ip_inc(p);
2533 }
2534
2535 static inline void
2536 instr_learner_exec(struct rte_swx_pipeline *p)
2537 {
2538         struct thread *t = &p->threads[p->thread_id];
2539         struct instruction *ip = t->ip;
2540         uint32_t learner_id = ip->table.table_id;
2541         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2542                 p->n_selectors + learner_id];
2543         struct learner_runtime *l = &t->learners[learner_id];
2544         struct learner_statistics *stats = &p->learner_stats[learner_id];
2545         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2546         uint8_t *action_data;
2547         int done, hit;
2548
2549         /* Table. */
2550         time = rte_get_tsc_cycles();
2551
2552         done = rte_swx_table_learner_lookup(ts->obj,
2553                                             l->mailbox,
2554                                             time,
2555                                             l->key,
2556                                             &action_id,
2557                                             &action_data,
2558                                             &hit);
2559         if (!done) {
2560                 /* Thread. */
2561                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2562                       p->thread_id,
2563                       learner_id);
2564
2565                 thread_yield(p);
2566                 return;
2567         }
2568
2569         action_id = hit ? action_id : ts->default_action_id;
2570         action_data = hit ? action_data : ts->default_action_data;
2571         n_pkts_hit = stats->n_pkts_hit[hit];
2572         n_pkts_action = stats->n_pkts_action[action_id];
2573
2574         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2575               p->thread_id,
2576               learner_id,
2577               hit ? "hit" : "miss",
2578               (uint32_t)action_id);
2579
2580         t->action_id = action_id;
2581         t->structs[0] = action_data;
2582         t->hit = hit;
2583         t->learner_id = learner_id;
2584         t->time = time;
2585         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2586         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2587
2588         /* Thread. */
2589         thread_ip_action_call(p, t, action_id);
2590 }
2591
2592 static inline void
2593 instr_learner_af_exec(struct rte_swx_pipeline *p)
2594 {
2595         struct thread *t = &p->threads[p->thread_id];
2596         struct instruction *ip = t->ip;
2597         uint32_t learner_id = ip->table.table_id;
2598         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2599                 p->n_selectors + learner_id];
2600         struct learner_runtime *l = &t->learners[learner_id];
2601         struct learner_statistics *stats = &p->learner_stats[learner_id];
2602         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2603         uint8_t *action_data;
2604         action_func_t action_func;
2605         int done, hit;
2606
2607         /* Table. */
2608         time = rte_get_tsc_cycles();
2609
2610         done = rte_swx_table_learner_lookup(ts->obj,
2611                                             l->mailbox,
2612                                             time,
2613                                             l->key,
2614                                             &action_id,
2615                                             &action_data,
2616                                             &hit);
2617         if (!done) {
2618                 /* Thread. */
2619                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2620                       p->thread_id,
2621                       learner_id);
2622
2623                 thread_yield(p);
2624                 return;
2625         }
2626
2627         action_id = hit ? action_id : ts->default_action_id;
2628         action_data = hit ? action_data : ts->default_action_data;
2629         action_func = p->action_funcs[action_id];
2630         n_pkts_hit = stats->n_pkts_hit[hit];
2631         n_pkts_action = stats->n_pkts_action[action_id];
2632
2633         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2634               p->thread_id,
2635               learner_id,
2636               hit ? "hit" : "miss",
2637               (uint32_t)action_id);
2638
2639         t->action_id = action_id;
2640         t->structs[0] = action_data;
2641         t->hit = hit;
2642         t->learner_id = learner_id;
2643         t->time = time;
2644         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2645         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2646
2647         /* Thread. */
2648         thread_ip_inc(p);
2649
2650         /* Action */
2651         action_func(p);
2652 }
2653
2654 /*
2655  * learn.
2656  */
2657 static struct action *
2658 action_find(struct rte_swx_pipeline *p, const char *name);
2659
2660 static int
2661 action_has_nbo_args(struct action *a);
2662
2663 static int
2664 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2665
2666 static int
2667 instr_learn_translate(struct rte_swx_pipeline *p,
2668                       struct action *action,
2669                       char **tokens,
2670                       int n_tokens,
2671                       struct instruction *instr,
2672                       struct instruction_data *data __rte_unused)
2673 {
2674         struct action *a;
2675         struct field *mf_first_arg = NULL, *mf_timeout_id = NULL;
2676         const char *mf_first_arg_name, *mf_timeout_id_name;
2677
2678         CHECK(action, EINVAL);
2679         CHECK((n_tokens == 3) || (n_tokens == 4), EINVAL);
2680
2681         /* Action. */
2682         a = action_find(p, tokens[1]);
2683         CHECK(a, EINVAL);
2684         CHECK(!action_has_nbo_args(a), EINVAL);
2685
2686         /* Action first argument. */
2687         mf_first_arg_name = (n_tokens == 4) ? tokens[2] : NULL;
2688         CHECK(!learner_action_args_check(p, a, mf_first_arg_name), EINVAL);
2689
2690         if (mf_first_arg_name) {
2691                 mf_first_arg = metadata_field_parse(p, mf_first_arg_name);
2692                 CHECK(mf_first_arg, EINVAL);
2693         }
2694
2695         /* Timeout ID. */
2696         mf_timeout_id_name = (n_tokens == 4) ? tokens[3] : tokens[2];
2697         CHECK_NAME(mf_timeout_id_name, EINVAL);
2698         mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
2699         CHECK(mf_timeout_id, EINVAL);
2700
2701         /* Instruction. */
2702         instr->type = INSTR_LEARNER_LEARN;
2703         instr->learn.action_id = a->id;
2704         instr->learn.mf_first_arg_offset = mf_first_arg ? (mf_first_arg->offset / 8) : 0;
2705         instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
2706         instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;
2707
2708         return 0;
2709 }
2710
2711 static inline void
2712 instr_learn_exec(struct rte_swx_pipeline *p)
2713 {
2714         struct thread *t = &p->threads[p->thread_id];
2715         struct instruction *ip = t->ip;
2716
2717         __instr_learn_exec(p, t, ip);
2718
2719         /* Thread. */
2720         thread_ip_inc(p);
2721 }
2722
2723 /*
2724  * rearm.
2725  */
2726 static int
2727 instr_rearm_translate(struct rte_swx_pipeline *p,
2728                       struct action *action,
2729                       char **tokens,
2730                       int n_tokens,
2731                       struct instruction *instr,
2732                       struct instruction_data *data __rte_unused)
2733 {
2734         struct field *mf_timeout_id;
2735         const char *mf_timeout_id_name;
2736
2737         CHECK(action, EINVAL);
2738         CHECK((n_tokens == 1) || (n_tokens == 2), EINVAL);
2739
2740         /* INSTR_LEARNER_REARM. */
2741         if (n_tokens == 1) {
2742                 instr->type = INSTR_LEARNER_REARM;
2743                 return 0;
2744         }
2745
2746         /* INSTR_LEARNER_REARM_NEW. */
2747         mf_timeout_id_name = tokens[1];
2748         CHECK_NAME(mf_timeout_id_name, EINVAL);
2749         mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
2750         CHECK(mf_timeout_id, EINVAL);
2751
2752         instr->type = INSTR_LEARNER_REARM_NEW;
2753         instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
2754         instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;
2755
2756         return 0;
2757 }
2758
2759 static inline void
2760 instr_rearm_exec(struct rte_swx_pipeline *p)
2761 {
2762         struct thread *t = &p->threads[p->thread_id];
2763         struct instruction *ip = t->ip;
2764
2765         __instr_rearm_exec(p, t, ip);
2766
2767         /* Thread. */
2768         thread_ip_inc(p);
2769 }
2770
2771 static inline void
2772 instr_rearm_new_exec(struct rte_swx_pipeline *p)
2773 {
2774         struct thread *t = &p->threads[p->thread_id];
2775         struct instruction *ip = t->ip;
2776
2777         __instr_rearm_new_exec(p, t, ip);
2778
2779         /* Thread. */
2780         thread_ip_inc(p);
2781 }
2782
2783 /*
2784  * forget.
2785  */
2786 static int
2787 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2788                        struct action *action,
2789                        char **tokens __rte_unused,
2790                        int n_tokens,
2791                        struct instruction *instr,
2792                        struct instruction_data *data __rte_unused)
2793 {
2794         CHECK(action, EINVAL);
2795         CHECK(n_tokens == 1, EINVAL);
2796
2797         instr->type = INSTR_LEARNER_FORGET;
2798
2799         return 0;
2800 }
2801
2802 static inline void
2803 instr_forget_exec(struct rte_swx_pipeline *p)
2804 {
2805         struct thread *t = &p->threads[p->thread_id];
2806         struct instruction *ip = t->ip;
2807
2808         __instr_forget_exec(p, t, ip);
2809
2810         /* Thread. */
2811         thread_ip_inc(p);
2812 }
2813
2814 /*
2815  * extern.
2816  */
2817 static int
2818 instr_extern_translate(struct rte_swx_pipeline *p,
2819                        struct action *action __rte_unused,
2820                        char **tokens,
2821                        int n_tokens,
2822                        struct instruction *instr,
2823                        struct instruction_data *data __rte_unused)
2824 {
2825         char *token = tokens[1];
2826
2827         CHECK(n_tokens == 2, EINVAL);
2828
2829         if (token[0] == 'e') {
2830                 struct extern_obj *obj;
2831                 struct extern_type_member_func *func;
2832
2833                 func = extern_obj_member_func_parse(p, token, &obj);
2834                 CHECK(func, EINVAL);
2835
2836                 instr->type = INSTR_EXTERN_OBJ;
2837                 instr->ext_obj.ext_obj_id = obj->id;
2838                 instr->ext_obj.func_id = func->id;
2839
2840                 return 0;
2841         }
2842
2843         if (token[0] == 'f') {
2844                 struct extern_func *func;
2845
2846                 func = extern_func_parse(p, token);
2847                 CHECK(func, EINVAL);
2848
2849                 instr->type = INSTR_EXTERN_FUNC;
2850                 instr->ext_func.ext_func_id = func->id;
2851
2852                 return 0;
2853         }
2854
2855         CHECK(0, EINVAL);
2856 }
2857
2858 static inline void
2859 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2860 {
2861         struct thread *t = &p->threads[p->thread_id];
2862         struct instruction *ip = t->ip;
2863         uint32_t done;
2864
2865         /* Extern object member function execute. */
2866         done = __instr_extern_obj_exec(p, t, ip);
2867
2868         /* Thread. */
2869         thread_ip_inc_cond(t, done);
2870         thread_yield_cond(p, done ^ 1);
2871 }
2872
2873 static inline void
2874 instr_extern_func_exec(struct rte_swx_pipeline *p)
2875 {
2876         struct thread *t = &p->threads[p->thread_id];
2877         struct instruction *ip = t->ip;
2878         uint32_t done;
2879
2880         /* Extern function execute. */
2881         done = __instr_extern_func_exec(p, t, ip);
2882
2883         /* Thread. */
2884         thread_ip_inc_cond(t, done);
2885         thread_yield_cond(p, done ^ 1);
2886 }
2887
2888 /*
2889  * hash.
2890  */
2891 static int
2892 instr_hash_translate(struct rte_swx_pipeline *p,
2893                      struct action *action,
2894                      char **tokens,
2895                      int n_tokens,
2896                      struct instruction *instr,
2897                      struct instruction_data *data __rte_unused)
2898 {
2899         struct hash_func *func;
2900         struct field *dst, *src_first, *src_last;
2901         uint32_t src_struct_id_first = 0, src_struct_id_last = 0;
2902
2903         CHECK(n_tokens == 5, EINVAL);
2904
2905         func = hash_func_find(p, tokens[1]);
2906         CHECK(func, EINVAL);
2907
2908         dst = metadata_field_parse(p, tokens[2]);
2909         CHECK(dst, EINVAL);
2910
2911         src_first = struct_field_parse(p, action, tokens[3], &src_struct_id_first);
2912         CHECK(src_first, EINVAL);
2913
2914         src_last = struct_field_parse(p, action, tokens[4], &src_struct_id_last);
2915         CHECK(src_last, EINVAL);
2916         CHECK(src_struct_id_first == src_struct_id_last, EINVAL);
2917
2918         instr->type = INSTR_HASH_FUNC;
2919         instr->hash_func.hash_func_id = (uint8_t)func->id;
2920         instr->hash_func.dst.offset = (uint8_t)dst->offset / 8;
2921         instr->hash_func.dst.n_bits = (uint8_t)dst->n_bits;
2922         instr->hash_func.src.struct_id = (uint8_t)src_struct_id_first;
2923         instr->hash_func.src.offset = (uint16_t)src_first->offset / 8;
2924         instr->hash_func.src.n_bytes = (uint16_t)((src_last->offset + src_last->n_bits -
2925                 src_first->offset) / 8);
2926
2927         return 0;
2928 }
2929
2930 static inline void
2931 instr_hash_func_exec(struct rte_swx_pipeline *p)
2932 {
2933         struct thread *t = &p->threads[p->thread_id];
2934         struct instruction *ip = t->ip;
2935
2936         /* Extern function execute. */
2937         __instr_hash_func_exec(p, t, ip);
2938
2939         /* Thread. */
2940         thread_ip_inc(p);
2941 }
2942
2943 /*
2944  * mov.
2945  */
2946 static int
2947 instr_mov_translate(struct rte_swx_pipeline *p,
2948                     struct action *action,
2949                     char **tokens,
2950                     int n_tokens,
2951                     struct instruction *instr,
2952                     struct instruction_data *data __rte_unused)
2953 {
2954         char *dst = tokens[1], *src = tokens[2];
2955         struct field *fdst, *fsrc;
2956         uint64_t src_val;
2957         uint32_t dst_struct_id = 0, src_struct_id = 0;
2958
2959         CHECK(n_tokens == 3, EINVAL);
2960
2961         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2962         CHECK(fdst, EINVAL);
2963         CHECK(!fdst->var_size, EINVAL);
2964
2965         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2966         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2967         if (fsrc) {
2968                 CHECK(!fsrc->var_size, EINVAL);
2969
2970                 instr->type = INSTR_MOV;
2971                 if (dst[0] != 'h' && src[0] == 'h')
2972                         instr->type = INSTR_MOV_MH;
2973                 if (dst[0] == 'h' && src[0] != 'h')
2974                         instr->type = INSTR_MOV_HM;
2975                 if (dst[0] == 'h' && src[0] == 'h')
2976                         instr->type = INSTR_MOV_HH;
2977
2978                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2979                 instr->mov.dst.n_bits = fdst->n_bits;
2980                 instr->mov.dst.offset = fdst->offset / 8;
2981                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2982                 instr->mov.src.n_bits = fsrc->n_bits;
2983                 instr->mov.src.offset = fsrc->offset / 8;
2984                 return 0;
2985         }
2986
2987         /* MOV_I. */
2988         src_val = strtoull(src, &src, 0);
2989         CHECK(!src[0], EINVAL);
2990
2991         if (dst[0] == 'h')
2992                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2993
2994         instr->type = INSTR_MOV_I;
2995         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2996         instr->mov.dst.n_bits = fdst->n_bits;
2997         instr->mov.dst.offset = fdst->offset / 8;
2998         instr->mov.src_val = src_val;
2999         return 0;
3000 }
3001
3002 static inline void
3003 instr_mov_exec(struct rte_swx_pipeline *p)
3004 {
3005         struct thread *t = &p->threads[p->thread_id];
3006         struct instruction *ip = t->ip;
3007
3008         __instr_mov_exec(p, t, ip);
3009
3010         /* Thread. */
3011         thread_ip_inc(p);
3012 }
3013
3014 static inline void
3015 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3016 {
3017         struct thread *t = &p->threads[p->thread_id];
3018         struct instruction *ip = t->ip;
3019
3020         __instr_mov_mh_exec(p, t, ip);
3021
3022         /* Thread. */
3023         thread_ip_inc(p);
3024 }
3025
3026 static inline void
3027 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3028 {
3029         struct thread *t = &p->threads[p->thread_id];
3030         struct instruction *ip = t->ip;
3031
3032         __instr_mov_hm_exec(p, t, ip);
3033
3034         /* Thread. */
3035         thread_ip_inc(p);
3036 }
3037
3038 static inline void
3039 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3040 {
3041         struct thread *t = &p->threads[p->thread_id];
3042         struct instruction *ip = t->ip;
3043
3044         __instr_mov_hh_exec(p, t, ip);
3045
3046         /* Thread. */
3047         thread_ip_inc(p);
3048 }
3049
3050 static inline void
3051 instr_mov_i_exec(struct rte_swx_pipeline *p)
3052 {
3053         struct thread *t = &p->threads[p->thread_id];
3054         struct instruction *ip = t->ip;
3055
3056         __instr_mov_i_exec(p, t, ip);
3057
3058         /* Thread. */
3059         thread_ip_inc(p);
3060 }
3061
3062 /*
3063  * dma.
3064  */
3065 static inline void
3066 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3067 {
3068         struct thread *t = &p->threads[p->thread_id];
3069         struct instruction *ip = t->ip;
3070
3071         __instr_dma_ht_exec(p, t, ip);
3072
3073         /* Thread. */
3074         thread_ip_inc(p);
3075 }
3076
3077 static inline void
3078 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3079 {
3080         struct thread *t = &p->threads[p->thread_id];
3081         struct instruction *ip = t->ip;
3082
3083         __instr_dma_ht2_exec(p, t, ip);
3084
3085         /* Thread. */
3086         thread_ip_inc(p);
3087 }
3088
3089 static inline void
3090 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3091 {
3092         struct thread *t = &p->threads[p->thread_id];
3093         struct instruction *ip = t->ip;
3094
3095         __instr_dma_ht3_exec(p, t, ip);
3096
3097         /* Thread. */
3098         thread_ip_inc(p);
3099 }
3100
3101 static inline void
3102 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3103 {
3104         struct thread *t = &p->threads[p->thread_id];
3105         struct instruction *ip = t->ip;
3106
3107         __instr_dma_ht4_exec(p, t, ip);
3108
3109         /* Thread. */
3110         thread_ip_inc(p);
3111 }
3112
3113 static inline void
3114 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3115 {
3116         struct thread *t = &p->threads[p->thread_id];
3117         struct instruction *ip = t->ip;
3118
3119         __instr_dma_ht5_exec(p, t, ip);
3120
3121         /* Thread. */
3122         thread_ip_inc(p);
3123 }
3124
3125 static inline void
3126 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3127 {
3128         struct thread *t = &p->threads[p->thread_id];
3129         struct instruction *ip = t->ip;
3130
3131         __instr_dma_ht6_exec(p, t, ip);
3132
3133         /* Thread. */
3134         thread_ip_inc(p);
3135 }
3136
3137 static inline void
3138 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3139 {
3140         struct thread *t = &p->threads[p->thread_id];
3141         struct instruction *ip = t->ip;
3142
3143         __instr_dma_ht7_exec(p, t, ip);
3144
3145         /* Thread. */
3146         thread_ip_inc(p);
3147 }
3148
3149 static inline void
3150 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3151 {
3152         struct thread *t = &p->threads[p->thread_id];
3153         struct instruction *ip = t->ip;
3154
3155         __instr_dma_ht8_exec(p, t, ip);
3156
3157         /* Thread. */
3158         thread_ip_inc(p);
3159 }
3160
3161 /*
3162  * alu.
3163  */
3164 static int
3165 instr_alu_add_translate(struct rte_swx_pipeline *p,
3166                         struct action *action,
3167                         char **tokens,
3168                         int n_tokens,
3169                         struct instruction *instr,
3170                         struct instruction_data *data __rte_unused)
3171 {
3172         char *dst = tokens[1], *src = tokens[2];
3173         struct field *fdst, *fsrc;
3174         uint64_t src_val;
3175         uint32_t dst_struct_id = 0, src_struct_id = 0;
3176
3177         CHECK(n_tokens == 3, EINVAL);
3178
3179         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3180         CHECK(fdst, EINVAL);
3181         CHECK(!fdst->var_size, EINVAL);
3182
3183         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3184         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3185         if (fsrc) {
3186                 CHECK(!fsrc->var_size, EINVAL);
3187
3188                 instr->type = INSTR_ALU_ADD;
3189                 if (dst[0] == 'h' && src[0] != 'h')
3190                         instr->type = INSTR_ALU_ADD_HM;
3191                 if (dst[0] != 'h' && src[0] == 'h')
3192                         instr->type = INSTR_ALU_ADD_MH;
3193                 if (dst[0] == 'h' && src[0] == 'h')
3194                         instr->type = INSTR_ALU_ADD_HH;
3195
3196                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3197                 instr->alu.dst.n_bits = fdst->n_bits;
3198                 instr->alu.dst.offset = fdst->offset / 8;
3199                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3200                 instr->alu.src.n_bits = fsrc->n_bits;
3201                 instr->alu.src.offset = fsrc->offset / 8;
3202                 return 0;
3203         }
3204
3205         /* ADD_MI, ADD_HI. */
3206         src_val = strtoull(src, &src, 0);
3207         CHECK(!src[0], EINVAL);
3208
3209         instr->type = INSTR_ALU_ADD_MI;
3210         if (dst[0] == 'h')
3211                 instr->type = INSTR_ALU_ADD_HI;
3212
3213         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3214         instr->alu.dst.n_bits = fdst->n_bits;
3215         instr->alu.dst.offset = fdst->offset / 8;
3216         instr->alu.src_val = src_val;
3217         return 0;
3218 }
3219
3220 static int
3221 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3222                         struct action *action,
3223                         char **tokens,
3224                         int n_tokens,
3225                         struct instruction *instr,
3226                         struct instruction_data *data __rte_unused)
3227 {
3228         char *dst = tokens[1], *src = tokens[2];
3229         struct field *fdst, *fsrc;
3230         uint64_t src_val;
3231         uint32_t dst_struct_id = 0, src_struct_id = 0;
3232
3233         CHECK(n_tokens == 3, EINVAL);
3234
3235         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3236         CHECK(fdst, EINVAL);
3237         CHECK(!fdst->var_size, EINVAL);
3238
3239         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3240         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3241         if (fsrc) {
3242                 CHECK(!fsrc->var_size, EINVAL);
3243
3244                 instr->type = INSTR_ALU_SUB;
3245                 if (dst[0] == 'h' && src[0] != 'h')
3246                         instr->type = INSTR_ALU_SUB_HM;
3247                 if (dst[0] != 'h' && src[0] == 'h')
3248                         instr->type = INSTR_ALU_SUB_MH;
3249                 if (dst[0] == 'h' && src[0] == 'h')
3250                         instr->type = INSTR_ALU_SUB_HH;
3251
3252                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3253                 instr->alu.dst.n_bits = fdst->n_bits;
3254                 instr->alu.dst.offset = fdst->offset / 8;
3255                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3256                 instr->alu.src.n_bits = fsrc->n_bits;
3257                 instr->alu.src.offset = fsrc->offset / 8;
3258                 return 0;
3259         }
3260
3261         /* SUB_MI, SUB_HI. */
3262         src_val = strtoull(src, &src, 0);
3263         CHECK(!src[0], EINVAL);
3264
3265         instr->type = INSTR_ALU_SUB_MI;
3266         if (dst[0] == 'h')
3267                 instr->type = INSTR_ALU_SUB_HI;
3268
3269         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3270         instr->alu.dst.n_bits = fdst->n_bits;
3271         instr->alu.dst.offset = fdst->offset / 8;
3272         instr->alu.src_val = src_val;
3273         return 0;
3274 }
3275
3276 static int
3277 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3278                           struct action *action __rte_unused,
3279                           char **tokens,
3280                           int n_tokens,
3281                           struct instruction *instr,
3282                           struct instruction_data *data __rte_unused)
3283 {
3284         char *dst = tokens[1], *src = tokens[2];
3285         struct header *hdst, *hsrc;
3286         struct field *fdst, *fsrc;
3287
3288         CHECK(n_tokens == 3, EINVAL);
3289
3290         fdst = header_field_parse(p, dst, &hdst);
3291         CHECK(fdst, EINVAL);
3292         CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3293
3294         /* CKADD_FIELD. */
3295         fsrc = header_field_parse(p, src, &hsrc);
3296         if (fsrc) {
3297                 CHECK(!fsrc->var_size, EINVAL);
3298
3299                 instr->type = INSTR_ALU_CKADD_FIELD;
3300                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3301                 instr->alu.dst.n_bits = fdst->n_bits;
3302                 instr->alu.dst.offset = fdst->offset / 8;
3303                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3304                 instr->alu.src.n_bits = fsrc->n_bits;
3305                 instr->alu.src.offset = fsrc->offset / 8;
3306                 return 0;
3307         }
3308
3309         /* CKADD_STRUCT, CKADD_STRUCT20. */
3310         hsrc = header_parse(p, src);
3311         CHECK(hsrc, EINVAL);
3312
3313         instr->type = INSTR_ALU_CKADD_STRUCT;
3314         if (!hsrc->st->var_size && ((hsrc->st->n_bits / 8) == 20))
3315                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3316
3317         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3318         instr->alu.dst.n_bits = fdst->n_bits;
3319         instr->alu.dst.offset = fdst->offset / 8;
3320         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3321         instr->alu.src.n_bits = (uint8_t)hsrc->id; /* The src header ID is stored here. */
3322         instr->alu.src.offset = 0; /* Unused. */
3323         return 0;
3324 }
3325
3326 static int
3327 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3328                           struct action *action __rte_unused,
3329                           char **tokens,
3330                           int n_tokens,
3331                           struct instruction *instr,
3332                           struct instruction_data *data __rte_unused)
3333 {
3334         char *dst = tokens[1], *src = tokens[2];
3335         struct header *hdst, *hsrc;
3336         struct field *fdst, *fsrc;
3337
3338         CHECK(n_tokens == 3, EINVAL);
3339
3340         fdst = header_field_parse(p, dst, &hdst);
3341         CHECK(fdst, EINVAL);
3342         CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3343
3344         fsrc = header_field_parse(p, src, &hsrc);
3345         CHECK(fsrc, EINVAL);
3346         CHECK(!fsrc->var_size, EINVAL);
3347
3348         instr->type = INSTR_ALU_CKSUB_FIELD;
3349         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3350         instr->alu.dst.n_bits = fdst->n_bits;
3351         instr->alu.dst.offset = fdst->offset / 8;
3352         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3353         instr->alu.src.n_bits = fsrc->n_bits;
3354         instr->alu.src.offset = fsrc->offset / 8;
3355         return 0;
3356 }
3357
3358 static int
3359 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3360                         struct action *action,
3361                         char **tokens,
3362                         int n_tokens,
3363                         struct instruction *instr,
3364                         struct instruction_data *data __rte_unused)
3365 {
3366         char *dst = tokens[1], *src = tokens[2];
3367         struct field *fdst, *fsrc;
3368         uint64_t src_val;
3369         uint32_t dst_struct_id = 0, src_struct_id = 0;
3370
3371         CHECK(n_tokens == 3, EINVAL);
3372
3373         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3374         CHECK(fdst, EINVAL);
3375         CHECK(!fdst->var_size, EINVAL);
3376
3377         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3378         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3379         if (fsrc) {
3380                 CHECK(!fsrc->var_size, EINVAL);
3381
3382                 instr->type = INSTR_ALU_SHL;
3383                 if (dst[0] == 'h' && src[0] != 'h')
3384                         instr->type = INSTR_ALU_SHL_HM;
3385                 if (dst[0] != 'h' && src[0] == 'h')
3386                         instr->type = INSTR_ALU_SHL_MH;
3387                 if (dst[0] == 'h' && src[0] == 'h')
3388                         instr->type = INSTR_ALU_SHL_HH;
3389
3390                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3391                 instr->alu.dst.n_bits = fdst->n_bits;
3392                 instr->alu.dst.offset = fdst->offset / 8;
3393                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3394                 instr->alu.src.n_bits = fsrc->n_bits;
3395                 instr->alu.src.offset = fsrc->offset / 8;
3396                 return 0;
3397         }
3398
3399         /* SHL_MI, SHL_HI. */
3400         src_val = strtoull(src, &src, 0);
3401         CHECK(!src[0], EINVAL);
3402
3403         instr->type = INSTR_ALU_SHL_MI;
3404         if (dst[0] == 'h')
3405                 instr->type = INSTR_ALU_SHL_HI;
3406
3407         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3408         instr->alu.dst.n_bits = fdst->n_bits;
3409         instr->alu.dst.offset = fdst->offset / 8;
3410         instr->alu.src_val = src_val;
3411         return 0;
3412 }
3413
3414 static int
3415 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3416                         struct action *action,
3417                         char **tokens,
3418                         int n_tokens,
3419                         struct instruction *instr,
3420                         struct instruction_data *data __rte_unused)
3421 {
3422         char *dst = tokens[1], *src = tokens[2];
3423         struct field *fdst, *fsrc;
3424         uint64_t src_val;
3425         uint32_t dst_struct_id = 0, src_struct_id = 0;
3426
3427         CHECK(n_tokens == 3, EINVAL);
3428
3429         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3430         CHECK(fdst, EINVAL);
3431         CHECK(!fdst->var_size, EINVAL);
3432
3433         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3434         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3435         if (fsrc) {
3436                 CHECK(!fsrc->var_size, EINVAL);
3437
3438                 instr->type = INSTR_ALU_SHR;
3439                 if (dst[0] == 'h' && src[0] != 'h')
3440                         instr->type = INSTR_ALU_SHR_HM;
3441                 if (dst[0] != 'h' && src[0] == 'h')
3442                         instr->type = INSTR_ALU_SHR_MH;
3443                 if (dst[0] == 'h' && src[0] == 'h')
3444                         instr->type = INSTR_ALU_SHR_HH;
3445
3446                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3447                 instr->alu.dst.n_bits = fdst->n_bits;
3448                 instr->alu.dst.offset = fdst->offset / 8;
3449                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3450                 instr->alu.src.n_bits = fsrc->n_bits;
3451                 instr->alu.src.offset = fsrc->offset / 8;
3452                 return 0;
3453         }
3454
3455         /* SHR_MI, SHR_HI. */
3456         src_val = strtoull(src, &src, 0);
3457         CHECK(!src[0], EINVAL);
3458
3459         instr->type = INSTR_ALU_SHR_MI;
3460         if (dst[0] == 'h')
3461                 instr->type = INSTR_ALU_SHR_HI;
3462
3463         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3464         instr->alu.dst.n_bits = fdst->n_bits;
3465         instr->alu.dst.offset = fdst->offset / 8;
3466         instr->alu.src_val = src_val;
3467         return 0;
3468 }
3469
3470 static int
3471 instr_alu_and_translate(struct rte_swx_pipeline *p,
3472                         struct action *action,
3473                         char **tokens,
3474                         int n_tokens,
3475                         struct instruction *instr,
3476                         struct instruction_data *data __rte_unused)
3477 {
3478         char *dst = tokens[1], *src = tokens[2];
3479         struct field *fdst, *fsrc;
3480         uint64_t src_val;
3481         uint32_t dst_struct_id = 0, src_struct_id = 0;
3482
3483         CHECK(n_tokens == 3, EINVAL);
3484
3485         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3486         CHECK(fdst, EINVAL);
3487         CHECK(!fdst->var_size, EINVAL);
3488
3489         /* AND, AND_MH, AND_HM, AND_HH. */
3490         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3491         if (fsrc) {
3492                 CHECK(!fsrc->var_size, EINVAL);
3493
3494                 instr->type = INSTR_ALU_AND;
3495                 if (dst[0] != 'h' && src[0] == 'h')
3496                         instr->type = INSTR_ALU_AND_MH;
3497                 if (dst[0] == 'h' && src[0] != 'h')
3498                         instr->type = INSTR_ALU_AND_HM;
3499                 if (dst[0] == 'h' && src[0] == 'h')
3500                         instr->type = INSTR_ALU_AND_HH;
3501
3502                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3503                 instr->alu.dst.n_bits = fdst->n_bits;
3504                 instr->alu.dst.offset = fdst->offset / 8;
3505                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3506                 instr->alu.src.n_bits = fsrc->n_bits;
3507                 instr->alu.src.offset = fsrc->offset / 8;
3508                 return 0;
3509         }
3510
3511         /* AND_I. */
3512         src_val = strtoull(src, &src, 0);
3513         CHECK(!src[0], EINVAL);
3514
3515         if (dst[0] == 'h')
3516                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3517
3518         instr->type = INSTR_ALU_AND_I;
3519         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3520         instr->alu.dst.n_bits = fdst->n_bits;
3521         instr->alu.dst.offset = fdst->offset / 8;
3522         instr->alu.src_val = src_val;
3523         return 0;
3524 }
3525
3526 static int
3527 instr_alu_or_translate(struct rte_swx_pipeline *p,
3528                        struct action *action,
3529                        char **tokens,
3530                        int n_tokens,
3531                        struct instruction *instr,
3532                        struct instruction_data *data __rte_unused)
3533 {
3534         char *dst = tokens[1], *src = tokens[2];
3535         struct field *fdst, *fsrc;
3536         uint64_t src_val;
3537         uint32_t dst_struct_id = 0, src_struct_id = 0;
3538
3539         CHECK(n_tokens == 3, EINVAL);
3540
3541         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3542         CHECK(fdst, EINVAL);
3543         CHECK(!fdst->var_size, EINVAL);
3544
3545         /* OR, OR_MH, OR_HM, OR_HH. */
3546         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3547         if (fsrc) {
3548                 CHECK(!fsrc->var_size, EINVAL);
3549
3550                 instr->type = INSTR_ALU_OR;
3551                 if (dst[0] != 'h' && src[0] == 'h')
3552                         instr->type = INSTR_ALU_OR_MH;
3553                 if (dst[0] == 'h' && src[0] != 'h')
3554                         instr->type = INSTR_ALU_OR_HM;
3555                 if (dst[0] == 'h' && src[0] == 'h')
3556                         instr->type = INSTR_ALU_OR_HH;
3557
3558                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3559                 instr->alu.dst.n_bits = fdst->n_bits;
3560                 instr->alu.dst.offset = fdst->offset / 8;
3561                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3562                 instr->alu.src.n_bits = fsrc->n_bits;
3563                 instr->alu.src.offset = fsrc->offset / 8;
3564                 return 0;
3565         }
3566
3567         /* OR_I. */
3568         src_val = strtoull(src, &src, 0);
3569         CHECK(!src[0], EINVAL);
3570
3571         if (dst[0] == 'h')
3572                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3573
3574         instr->type = INSTR_ALU_OR_I;
3575         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3576         instr->alu.dst.n_bits = fdst->n_bits;
3577         instr->alu.dst.offset = fdst->offset / 8;
3578         instr->alu.src_val = src_val;
3579         return 0;
3580 }
3581
3582 static int
3583 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3584                         struct action *action,
3585                         char **tokens,
3586                         int n_tokens,
3587                         struct instruction *instr,
3588                         struct instruction_data *data __rte_unused)
3589 {
3590         char *dst = tokens[1], *src = tokens[2];
3591         struct field *fdst, *fsrc;
3592         uint64_t src_val;
3593         uint32_t dst_struct_id = 0, src_struct_id = 0;
3594
3595         CHECK(n_tokens == 3, EINVAL);
3596
3597         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3598         CHECK(fdst, EINVAL);
3599         CHECK(!fdst->var_size, EINVAL);
3600
3601         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3602         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3603         if (fsrc) {
3604                 CHECK(!fsrc->var_size, EINVAL);
3605
3606                 instr->type = INSTR_ALU_XOR;
3607                 if (dst[0] != 'h' && src[0] == 'h')
3608                         instr->type = INSTR_ALU_XOR_MH;
3609                 if (dst[0] == 'h' && src[0] != 'h')
3610                         instr->type = INSTR_ALU_XOR_HM;
3611                 if (dst[0] == 'h' && src[0] == 'h')
3612                         instr->type = INSTR_ALU_XOR_HH;
3613
3614                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3615                 instr->alu.dst.n_bits = fdst->n_bits;
3616                 instr->alu.dst.offset = fdst->offset / 8;
3617                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3618                 instr->alu.src.n_bits = fsrc->n_bits;
3619                 instr->alu.src.offset = fsrc->offset / 8;
3620                 return 0;
3621         }
3622
3623         /* XOR_I. */
3624         src_val = strtoull(src, &src, 0);
3625         CHECK(!src[0], EINVAL);
3626
3627         if (dst[0] == 'h')
3628                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3629
3630         instr->type = INSTR_ALU_XOR_I;
3631         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3632         instr->alu.dst.n_bits = fdst->n_bits;
3633         instr->alu.dst.offset = fdst->offset / 8;
3634         instr->alu.src_val = src_val;
3635         return 0;
3636 }
3637
3638 static inline void
3639 instr_alu_add_exec(struct rte_swx_pipeline *p)
3640 {
3641         struct thread *t = &p->threads[p->thread_id];
3642         struct instruction *ip = t->ip;
3643
3644         /* Structs */
3645         __instr_alu_add_exec(p, t, ip);
3646
3647         /* Thread. */
3648         thread_ip_inc(p);
3649 }
3650
3651 static inline void
3652 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3653 {
3654         struct thread *t = &p->threads[p->thread_id];
3655         struct instruction *ip = t->ip;
3656
3657         /* Structs. */
3658         __instr_alu_add_mh_exec(p, t, ip);
3659
3660         /* Thread. */
3661         thread_ip_inc(p);
3662 }
3663
3664 static inline void
3665 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3666 {
3667         struct thread *t = &p->threads[p->thread_id];
3668         struct instruction *ip = t->ip;
3669
3670         /* Structs. */
3671         __instr_alu_add_hm_exec(p, t, ip);
3672
3673         /* Thread. */
3674         thread_ip_inc(p);
3675 }
3676
3677 static inline void
3678 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3679 {
3680         struct thread *t = &p->threads[p->thread_id];
3681         struct instruction *ip = t->ip;
3682
3683         /* Structs. */
3684         __instr_alu_add_hh_exec(p, t, ip);
3685
3686         /* Thread. */
3687         thread_ip_inc(p);
3688 }
3689
3690 static inline void
3691 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3692 {
3693         struct thread *t = &p->threads[p->thread_id];
3694         struct instruction *ip = t->ip;
3695
3696         /* Structs. */
3697         __instr_alu_add_mi_exec(p, t, ip);
3698
3699         /* Thread. */
3700         thread_ip_inc(p);
3701 }
3702
3703 static inline void
3704 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3705 {
3706         struct thread *t = &p->threads[p->thread_id];
3707         struct instruction *ip = t->ip;
3708
3709         /* Structs. */
3710         __instr_alu_add_hi_exec(p, t, ip);
3711
3712         /* Thread. */
3713         thread_ip_inc(p);
3714 }
3715
3716 static inline void
3717 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3718 {
3719         struct thread *t = &p->threads[p->thread_id];
3720         struct instruction *ip = t->ip;
3721
3722         /* Structs. */
3723         __instr_alu_sub_exec(p, t, ip);
3724
3725         /* Thread. */
3726         thread_ip_inc(p);
3727 }
3728
3729 static inline void
3730 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3731 {
3732         struct thread *t = &p->threads[p->thread_id];
3733         struct instruction *ip = t->ip;
3734
3735         /* Structs. */
3736         __instr_alu_sub_mh_exec(p, t, ip);
3737
3738         /* Thread. */
3739         thread_ip_inc(p);
3740 }
3741
3742 static inline void
3743 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3744 {
3745         struct thread *t = &p->threads[p->thread_id];
3746         struct instruction *ip = t->ip;
3747
3748         /* Structs. */
3749         __instr_alu_sub_hm_exec(p, t, ip);
3750
3751         /* Thread. */
3752         thread_ip_inc(p);
3753 }
3754
3755 static inline void
3756 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3757 {
3758         struct thread *t = &p->threads[p->thread_id];
3759         struct instruction *ip = t->ip;
3760
3761         /* Structs. */
3762         __instr_alu_sub_hh_exec(p, t, ip);
3763
3764         /* Thread. */
3765         thread_ip_inc(p);
3766 }
3767
3768 static inline void
3769 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3770 {
3771         struct thread *t = &p->threads[p->thread_id];
3772         struct instruction *ip = t->ip;
3773
3774         /* Structs. */
3775         __instr_alu_sub_mi_exec(p, t, ip);
3776
3777         /* Thread. */
3778         thread_ip_inc(p);
3779 }
3780
3781 static inline void
3782 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3783 {
3784         struct thread *t = &p->threads[p->thread_id];
3785         struct instruction *ip = t->ip;
3786
3787         /* Structs. */
3788         __instr_alu_sub_hi_exec(p, t, ip);
3789
3790         /* Thread. */
3791         thread_ip_inc(p);
3792 }
3793
3794 static inline void
3795 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3796 {
3797         struct thread *t = &p->threads[p->thread_id];
3798         struct instruction *ip = t->ip;
3799
3800         /* Structs. */
3801         __instr_alu_shl_exec(p, t, ip);
3802
3803         /* Thread. */
3804         thread_ip_inc(p);
3805 }
3806
3807 static inline void
3808 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3809 {
3810         struct thread *t = &p->threads[p->thread_id];
3811         struct instruction *ip = t->ip;
3812
3813         /* Structs. */
3814         __instr_alu_shl_mh_exec(p, t, ip);
3815
3816         /* Thread. */
3817         thread_ip_inc(p);
3818 }
3819
3820 static inline void
3821 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3822 {
3823         struct thread *t = &p->threads[p->thread_id];
3824         struct instruction *ip = t->ip;
3825
3826         /* Structs. */
3827         __instr_alu_shl_hm_exec(p, t, ip);
3828
3829         /* Thread. */
3830         thread_ip_inc(p);
3831 }
3832
3833 static inline void
3834 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3835 {
3836         struct thread *t = &p->threads[p->thread_id];
3837         struct instruction *ip = t->ip;
3838
3839         /* Structs. */
3840         __instr_alu_shl_hh_exec(p, t, ip);
3841
3842         /* Thread. */
3843         thread_ip_inc(p);
3844 }
3845
3846 static inline void
3847 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3848 {
3849         struct thread *t = &p->threads[p->thread_id];
3850         struct instruction *ip = t->ip;
3851
3852         /* Structs. */
3853         __instr_alu_shl_mi_exec(p, t, ip);
3854
3855         /* Thread. */
3856         thread_ip_inc(p);
3857 }
3858
3859 static inline void
3860 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3861 {
3862         struct thread *t = &p->threads[p->thread_id];
3863         struct instruction *ip = t->ip;
3864
3865         /* Structs. */
3866         __instr_alu_shl_hi_exec(p, t, ip);
3867
3868         /* Thread. */
3869         thread_ip_inc(p);
3870 }
3871
3872 static inline void
3873 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3874 {
3875         struct thread *t = &p->threads[p->thread_id];
3876         struct instruction *ip = t->ip;
3877
3878         /* Structs. */
3879         __instr_alu_shr_exec(p, t, ip);
3880
3881         /* Thread. */
3882         thread_ip_inc(p);
3883 }
3884
3885 static inline void
3886 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3887 {
3888         struct thread *t = &p->threads[p->thread_id];
3889         struct instruction *ip = t->ip;
3890
3891         /* Structs. */
3892         __instr_alu_shr_mh_exec(p, t, ip);
3893
3894         /* Thread. */
3895         thread_ip_inc(p);
3896 }
3897
3898 static inline void
3899 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3900 {
3901         struct thread *t = &p->threads[p->thread_id];
3902         struct instruction *ip = t->ip;
3903
3904         /* Structs. */
3905         __instr_alu_shr_hm_exec(p, t, ip);
3906
3907         /* Thread. */
3908         thread_ip_inc(p);
3909 }
3910
3911 static inline void
3912 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3913 {
3914         struct thread *t = &p->threads[p->thread_id];
3915         struct instruction *ip = t->ip;
3916
3917         /* Structs. */
3918         __instr_alu_shr_hh_exec(p, t, ip);
3919
3920         /* Thread. */
3921         thread_ip_inc(p);
3922 }
3923
3924 static inline void
3925 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3926 {
3927         struct thread *t = &p->threads[p->thread_id];
3928         struct instruction *ip = t->ip;
3929
3930         /* Structs. */
3931         __instr_alu_shr_mi_exec(p, t, ip);
3932
3933         /* Thread. */
3934         thread_ip_inc(p);
3935 }
3936
3937 static inline void
3938 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3939 {
3940         struct thread *t = &p->threads[p->thread_id];
3941         struct instruction *ip = t->ip;
3942
3943         /* Structs. */
3944         __instr_alu_shr_hi_exec(p, t, ip);
3945
3946         /* Thread. */
3947         thread_ip_inc(p);
3948 }
3949
3950 static inline void
3951 instr_alu_and_exec(struct rte_swx_pipeline *p)
3952 {
3953         struct thread *t = &p->threads[p->thread_id];
3954         struct instruction *ip = t->ip;
3955
3956         /* Structs. */
3957         __instr_alu_and_exec(p, t, ip);
3958
3959         /* Thread. */
3960         thread_ip_inc(p);
3961 }
3962
3963 static inline void
3964 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3965 {
3966         struct thread *t = &p->threads[p->thread_id];
3967         struct instruction *ip = t->ip;
3968
3969         /* Structs. */
3970         __instr_alu_and_mh_exec(p, t, ip);
3971
3972         /* Thread. */
3973         thread_ip_inc(p);
3974 }
3975
3976 static inline void
3977 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3978 {
3979         struct thread *t = &p->threads[p->thread_id];
3980         struct instruction *ip = t->ip;
3981
3982         /* Structs. */
3983         __instr_alu_and_hm_exec(p, t, ip);
3984
3985         /* Thread. */
3986         thread_ip_inc(p);
3987 }
3988
3989 static inline void
3990 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3991 {
3992         struct thread *t = &p->threads[p->thread_id];
3993         struct instruction *ip = t->ip;
3994
3995         /* Structs. */
3996         __instr_alu_and_hh_exec(p, t, ip);
3997
3998         /* Thread. */
3999         thread_ip_inc(p);
4000 }
4001
4002 static inline void
4003 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4004 {
4005         struct thread *t = &p->threads[p->thread_id];
4006         struct instruction *ip = t->ip;
4007
4008         /* Structs. */
4009         __instr_alu_and_i_exec(p, t, ip);
4010
4011         /* Thread. */
4012         thread_ip_inc(p);
4013 }
4014
4015 static inline void
4016 instr_alu_or_exec(struct rte_swx_pipeline *p)
4017 {
4018         struct thread *t = &p->threads[p->thread_id];
4019         struct instruction *ip = t->ip;
4020
4021         /* Structs. */
4022         __instr_alu_or_exec(p, t, ip);
4023
4024         /* Thread. */
4025         thread_ip_inc(p);
4026 }
4027
4028 static inline void
4029 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
4030 {
4031         struct thread *t = &p->threads[p->thread_id];
4032         struct instruction *ip = t->ip;
4033
4034         /* Structs. */
4035         __instr_alu_or_mh_exec(p, t, ip);
4036
4037         /* Thread. */
4038         thread_ip_inc(p);
4039 }
4040
4041 static inline void
4042 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
4043 {
4044         struct thread *t = &p->threads[p->thread_id];
4045         struct instruction *ip = t->ip;
4046
4047         /* Structs. */
4048         __instr_alu_or_hm_exec(p, t, ip);
4049
4050         /* Thread. */
4051         thread_ip_inc(p);
4052 }
4053
4054 static inline void
4055 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
4056 {
4057         struct thread *t = &p->threads[p->thread_id];
4058         struct instruction *ip = t->ip;
4059
4060         /* Structs. */
4061         __instr_alu_or_hh_exec(p, t, ip);
4062
4063         /* Thread. */
4064         thread_ip_inc(p);
4065 }
4066
4067 static inline void
4068 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4069 {
4070         struct thread *t = &p->threads[p->thread_id];
4071         struct instruction *ip = t->ip;
4072
4073         /* Structs. */
4074         __instr_alu_or_i_exec(p, t, ip);
4075
4076         /* Thread. */
4077         thread_ip_inc(p);
4078 }
4079
4080 static inline void
4081 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4082 {
4083         struct thread *t = &p->threads[p->thread_id];
4084         struct instruction *ip = t->ip;
4085
4086         /* Structs. */
4087         __instr_alu_xor_exec(p, t, ip);
4088
4089         /* Thread. */
4090         thread_ip_inc(p);
4091 }
4092
4093 static inline void
4094 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
4095 {
4096         struct thread *t = &p->threads[p->thread_id];
4097         struct instruction *ip = t->ip;
4098
4099         /* Structs. */
4100         __instr_alu_xor_mh_exec(p, t, ip);
4101
4102         /* Thread. */
4103         thread_ip_inc(p);
4104 }
4105
4106 static inline void
4107 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
4108 {
4109         struct thread *t = &p->threads[p->thread_id];
4110         struct instruction *ip = t->ip;
4111
4112         /* Structs. */
4113         __instr_alu_xor_hm_exec(p, t, ip);
4114
4115         /* Thread. */
4116         thread_ip_inc(p);
4117 }
4118
4119 static inline void
4120 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
4121 {
4122         struct thread *t = &p->threads[p->thread_id];
4123         struct instruction *ip = t->ip;
4124
4125         /* Structs. */
4126         __instr_alu_xor_hh_exec(p, t, ip);
4127
4128         /* Thread. */
4129         thread_ip_inc(p);
4130 }
4131
4132 static inline void
4133 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4134 {
4135         struct thread *t = &p->threads[p->thread_id];
4136         struct instruction *ip = t->ip;
4137
4138         /* Structs. */
4139         __instr_alu_xor_i_exec(p, t, ip);
4140
4141         /* Thread. */
4142         thread_ip_inc(p);
4143 }
4144
4145 static inline void
4146 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4147 {
4148         struct thread *t = &p->threads[p->thread_id];
4149         struct instruction *ip = t->ip;
4150
4151         /* Structs. */
4152         __instr_alu_ckadd_field_exec(p, t, ip);
4153
4154         /* Thread. */
4155         thread_ip_inc(p);
4156 }
4157
4158 static inline void
4159 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4160 {
4161         struct thread *t = &p->threads[p->thread_id];
4162         struct instruction *ip = t->ip;
4163
4164         /* Structs. */
4165         __instr_alu_cksub_field_exec(p, t, ip);
4166
4167         /* Thread. */
4168         thread_ip_inc(p);
4169 }
4170
4171 static inline void
4172 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4173 {
4174         struct thread *t = &p->threads[p->thread_id];
4175         struct instruction *ip = t->ip;
4176
4177         /* Structs. */
4178         __instr_alu_ckadd_struct20_exec(p, t, ip);
4179
4180         /* Thread. */
4181         thread_ip_inc(p);
4182 }
4183
4184 static inline void
4185 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4186 {
4187         struct thread *t = &p->threads[p->thread_id];
4188         struct instruction *ip = t->ip;
4189
4190         /* Structs. */
4191         __instr_alu_ckadd_struct_exec(p, t, ip);
4192
4193         /* Thread. */
4194         thread_ip_inc(p);
4195 }
4196
4197 /*
4198  * Register array.
4199  */
4200 static struct regarray *
4201 regarray_find(struct rte_swx_pipeline *p, const char *name);
4202
4203 static int
4204 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4205                       struct action *action,
4206                       char **tokens,
4207                       int n_tokens,
4208                       struct instruction *instr,
4209                       struct instruction_data *data __rte_unused)
4210 {
4211         char *regarray = tokens[1], *idx = tokens[2];
4212         struct regarray *r;
4213         struct field *fidx;
4214         uint32_t idx_struct_id, idx_val;
4215
4216         CHECK(n_tokens == 3, EINVAL);
4217
4218         r = regarray_find(p, regarray);
4219         CHECK(r, EINVAL);
4220
4221         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4222         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4223         if (fidx) {
4224                 CHECK(!fidx->var_size, EINVAL);
4225
4226                 instr->type = INSTR_REGPREFETCH_RM;
4227                 if (idx[0] == 'h')
4228                         instr->type = INSTR_REGPREFETCH_RH;
4229
4230                 instr->regarray.regarray_id = r->id;
4231                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4232                 instr->regarray.idx.n_bits = fidx->n_bits;
4233                 instr->regarray.idx.offset = fidx->offset / 8;
4234                 instr->regarray.dstsrc_val = 0; /* Unused. */
4235                 return 0;
4236         }
4237
4238         /* REGPREFETCH_RI. */
4239         idx_val = strtoul(idx, &idx, 0);
4240         CHECK(!idx[0], EINVAL);
4241
4242         instr->type = INSTR_REGPREFETCH_RI;
4243         instr->regarray.regarray_id = r->id;
4244         instr->regarray.idx_val = idx_val;
4245         instr->regarray.dstsrc_val = 0; /* Unused. */
4246         return 0;
4247 }
4248
4249 static int
4250 instr_regrd_translate(struct rte_swx_pipeline *p,
4251                       struct action *action,
4252                       char **tokens,
4253                       int n_tokens,
4254                       struct instruction *instr,
4255                       struct instruction_data *data __rte_unused)
4256 {
4257         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4258         struct regarray *r;
4259         struct field *fdst, *fidx;
4260         uint32_t dst_struct_id, idx_struct_id, idx_val;
4261
4262         CHECK(n_tokens == 4, EINVAL);
4263
4264         r = regarray_find(p, regarray);
4265         CHECK(r, EINVAL);
4266
4267         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4268         CHECK(fdst, EINVAL);
4269         CHECK(!fdst->var_size, EINVAL);
4270
4271         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4272         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4273         if (fidx) {
4274                 CHECK(!fidx->var_size, EINVAL);
4275
4276                 instr->type = INSTR_REGRD_MRM;
4277                 if (dst[0] == 'h' && idx[0] != 'h')
4278                         instr->type = INSTR_REGRD_HRM;
4279                 if (dst[0] != 'h' && idx[0] == 'h')
4280                         instr->type = INSTR_REGRD_MRH;
4281                 if (dst[0] == 'h' && idx[0] == 'h')
4282                         instr->type = INSTR_REGRD_HRH;
4283
4284                 instr->regarray.regarray_id = r->id;
4285                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4286                 instr->regarray.idx.n_bits = fidx->n_bits;
4287                 instr->regarray.idx.offset = fidx->offset / 8;
4288                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4289                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4290                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4291                 return 0;
4292         }
4293
4294         /* REGRD_MRI, REGRD_HRI. */
4295         idx_val = strtoul(idx, &idx, 0);
4296         CHECK(!idx[0], EINVAL);
4297
4298         instr->type = INSTR_REGRD_MRI;
4299         if (dst[0] == 'h')
4300                 instr->type = INSTR_REGRD_HRI;
4301
4302         instr->regarray.regarray_id = r->id;
4303         instr->regarray.idx_val = idx_val;
4304         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4305         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4306         instr->regarray.dstsrc.offset = fdst->offset / 8;
4307         return 0;
4308 }
4309
4310 static int
4311 instr_regwr_translate(struct rte_swx_pipeline *p,
4312                       struct action *action,
4313                       char **tokens,
4314                       int n_tokens,
4315                       struct instruction *instr,
4316                       struct instruction_data *data __rte_unused)
4317 {
4318         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4319         struct regarray *r;
4320         struct field *fidx, *fsrc;
4321         uint64_t src_val;
4322         uint32_t idx_struct_id, idx_val, src_struct_id;
4323
4324         CHECK(n_tokens == 4, EINVAL);
4325
4326         r = regarray_find(p, regarray);
4327         CHECK(r, EINVAL);
4328
4329         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4330         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4331         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4332         if (fidx && fsrc) {
4333                 CHECK(!fidx->var_size, EINVAL);
4334                 CHECK(!fsrc->var_size, EINVAL);
4335
4336                 instr->type = INSTR_REGWR_RMM;
4337                 if (idx[0] == 'h' && src[0] != 'h')
4338                         instr->type = INSTR_REGWR_RHM;
4339                 if (idx[0] != 'h' && src[0] == 'h')
4340                         instr->type = INSTR_REGWR_RMH;
4341                 if (idx[0] == 'h' && src[0] == 'h')
4342                         instr->type = INSTR_REGWR_RHH;
4343
4344                 instr->regarray.regarray_id = r->id;
4345                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4346                 instr->regarray.idx.n_bits = fidx->n_bits;
4347                 instr->regarray.idx.offset = fidx->offset / 8;
4348                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4349                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4350                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4351                 return 0;
4352         }
4353
4354         /* REGWR_RHI, REGWR_RMI. */
4355         if (fidx && !fsrc) {
4356                 CHECK(!fidx->var_size, EINVAL);
4357
4358                 src_val = strtoull(src, &src, 0);
4359                 CHECK(!src[0], EINVAL);
4360
4361                 instr->type = INSTR_REGWR_RMI;
4362                 if (idx[0] == 'h')
4363                         instr->type = INSTR_REGWR_RHI;
4364
4365                 instr->regarray.regarray_id = r->id;
4366                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4367                 instr->regarray.idx.n_bits = fidx->n_bits;
4368                 instr->regarray.idx.offset = fidx->offset / 8;
4369                 instr->regarray.dstsrc_val = src_val;
4370                 return 0;
4371         }
4372
4373         /* REGWR_RIH, REGWR_RIM. */
4374         if (!fidx && fsrc) {
4375                 idx_val = strtoul(idx, &idx, 0);
4376                 CHECK(!idx[0], EINVAL);
4377
4378                 CHECK(!fsrc->var_size, EINVAL);
4379
4380                 instr->type = INSTR_REGWR_RIM;
4381                 if (src[0] == 'h')
4382                         instr->type = INSTR_REGWR_RIH;
4383
4384                 instr->regarray.regarray_id = r->id;
4385                 instr->regarray.idx_val = idx_val;
4386                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4387                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4388                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4389                 return 0;
4390         }
4391
4392         /* REGWR_RII. */
4393         src_val = strtoull(src, &src, 0);
4394         CHECK(!src[0], EINVAL);
4395
4396         idx_val = strtoul(idx, &idx, 0);
4397         CHECK(!idx[0], EINVAL);
4398
4399         instr->type = INSTR_REGWR_RII;
4400         instr->regarray.idx_val = idx_val;
4401         instr->regarray.dstsrc_val = src_val;
4402
4403         return 0;
4404 }
4405
4406 static int
4407 instr_regadd_translate(struct rte_swx_pipeline *p,
4408                        struct action *action,
4409                        char **tokens,
4410                        int n_tokens,
4411                        struct instruction *instr,
4412                        struct instruction_data *data __rte_unused)
4413 {
4414         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4415         struct regarray *r;
4416         struct field *fidx, *fsrc;
4417         uint64_t src_val;
4418         uint32_t idx_struct_id, idx_val, src_struct_id;
4419
4420         CHECK(n_tokens == 4, EINVAL);
4421
4422         r = regarray_find(p, regarray);
4423         CHECK(r, EINVAL);
4424
4425         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4426         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4427         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4428         if (fidx && fsrc) {
4429                 CHECK(!fidx->var_size, EINVAL);
4430                 CHECK(!fsrc->var_size, EINVAL);
4431
4432                 instr->type = INSTR_REGADD_RMM;
4433                 if (idx[0] == 'h' && src[0] != 'h')
4434                         instr->type = INSTR_REGADD_RHM;
4435                 if (idx[0] != 'h' && src[0] == 'h')
4436                         instr->type = INSTR_REGADD_RMH;
4437                 if (idx[0] == 'h' && src[0] == 'h')
4438                         instr->type = INSTR_REGADD_RHH;
4439
4440                 instr->regarray.regarray_id = r->id;
4441                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4442                 instr->regarray.idx.n_bits = fidx->n_bits;
4443                 instr->regarray.idx.offset = fidx->offset / 8;
4444                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4445                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4446                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4447                 return 0;
4448         }
4449
4450         /* REGADD_RHI, REGADD_RMI. */
4451         if (fidx && !fsrc) {
4452                 CHECK(!fidx->var_size, EINVAL);
4453
4454                 src_val = strtoull(src, &src, 0);
4455                 CHECK(!src[0], EINVAL);
4456
4457                 instr->type = INSTR_REGADD_RMI;
4458                 if (idx[0] == 'h')
4459                         instr->type = INSTR_REGADD_RHI;
4460
4461                 instr->regarray.regarray_id = r->id;
4462                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4463                 instr->regarray.idx.n_bits = fidx->n_bits;
4464                 instr->regarray.idx.offset = fidx->offset / 8;
4465                 instr->regarray.dstsrc_val = src_val;
4466                 return 0;
4467         }
4468
4469         /* REGADD_RIH, REGADD_RIM. */
4470         if (!fidx && fsrc) {
4471                 idx_val = strtoul(idx, &idx, 0);
4472                 CHECK(!idx[0], EINVAL);
4473
4474                 CHECK(!fsrc->var_size, EINVAL);
4475
4476                 instr->type = INSTR_REGADD_RIM;
4477                 if (src[0] == 'h')
4478                         instr->type = INSTR_REGADD_RIH;
4479
4480                 instr->regarray.regarray_id = r->id;
4481                 instr->regarray.idx_val = idx_val;
4482                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4483                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4484                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4485                 return 0;
4486         }
4487
4488         /* REGADD_RII. */
4489         src_val = strtoull(src, &src, 0);
4490         CHECK(!src[0], EINVAL);
4491
4492         idx_val = strtoul(idx, &idx, 0);
4493         CHECK(!idx[0], EINVAL);
4494
4495         instr->type = INSTR_REGADD_RII;
4496         instr->regarray.idx_val = idx_val;
4497         instr->regarray.dstsrc_val = src_val;
4498         return 0;
4499 }
4500
4501 static inline void
4502 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4503 {
4504         struct thread *t = &p->threads[p->thread_id];
4505         struct instruction *ip = t->ip;
4506
4507         /* Structs. */
4508         __instr_regprefetch_rh_exec(p, t, ip);
4509
4510         /* Thread. */
4511         thread_ip_inc(p);
4512 }
4513
4514 static inline void
4515 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4516 {
4517         struct thread *t = &p->threads[p->thread_id];
4518         struct instruction *ip = t->ip;
4519
4520         /* Structs. */
4521         __instr_regprefetch_rm_exec(p, t, ip);
4522
4523         /* Thread. */
4524         thread_ip_inc(p);
4525 }
4526
4527 static inline void
4528 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4529 {
4530         struct thread *t = &p->threads[p->thread_id];
4531         struct instruction *ip = t->ip;
4532
4533         /* Structs. */
4534         __instr_regprefetch_ri_exec(p, t, ip);
4535
4536         /* Thread. */
4537         thread_ip_inc(p);
4538 }
4539
4540 static inline void
4541 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4542 {
4543         struct thread *t = &p->threads[p->thread_id];
4544         struct instruction *ip = t->ip;
4545
4546         /* Structs. */
4547         __instr_regrd_hrh_exec(p, t, ip);
4548
4549         /* Thread. */
4550         thread_ip_inc(p);
4551 }
4552
4553 static inline void
4554 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4555 {
4556         struct thread *t = &p->threads[p->thread_id];
4557         struct instruction *ip = t->ip;
4558
4559         /* Structs. */
4560         __instr_regrd_hrm_exec(p, t, ip);
4561
4562         /* Thread. */
4563         thread_ip_inc(p);
4564 }
4565
4566 static inline void
4567 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4568 {
4569         struct thread *t = &p->threads[p->thread_id];
4570         struct instruction *ip = t->ip;
4571
4572         /* Structs. */
4573         __instr_regrd_mrh_exec(p, t, ip);
4574
4575         /* Thread. */
4576         thread_ip_inc(p);
4577 }
4578
4579 static inline void
4580 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4581 {
4582         struct thread *t = &p->threads[p->thread_id];
4583         struct instruction *ip = t->ip;
4584
4585         /* Structs. */
4586         __instr_regrd_mrm_exec(p, t, ip);
4587
4588         /* Thread. */
4589         thread_ip_inc(p);
4590 }
4591
4592 static inline void
4593 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4594 {
4595         struct thread *t = &p->threads[p->thread_id];
4596         struct instruction *ip = t->ip;
4597
4598         /* Structs. */
4599         __instr_regrd_hri_exec(p, t, ip);
4600
4601         /* Thread. */
4602         thread_ip_inc(p);
4603 }
4604
4605 static inline void
4606 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4607 {
4608         struct thread *t = &p->threads[p->thread_id];
4609         struct instruction *ip = t->ip;
4610
4611         /* Structs. */
4612         __instr_regrd_mri_exec(p, t, ip);
4613
4614         /* Thread. */
4615         thread_ip_inc(p);
4616 }
4617
4618 static inline void
4619 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4620 {
4621         struct thread *t = &p->threads[p->thread_id];
4622         struct instruction *ip = t->ip;
4623
4624         /* Structs. */
4625         __instr_regwr_rhh_exec(p, t, ip);
4626
4627         /* Thread. */
4628         thread_ip_inc(p);
4629 }
4630
4631 static inline void
4632 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4633 {
4634         struct thread *t = &p->threads[p->thread_id];
4635         struct instruction *ip = t->ip;
4636
4637         /* Structs. */
4638         __instr_regwr_rhm_exec(p, t, ip);
4639
4640         /* Thread. */
4641         thread_ip_inc(p);
4642 }
4643
4644 static inline void
4645 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4646 {
4647         struct thread *t = &p->threads[p->thread_id];
4648         struct instruction *ip = t->ip;
4649
4650         /* Structs. */
4651         __instr_regwr_rmh_exec(p, t, ip);
4652
4653         /* Thread. */
4654         thread_ip_inc(p);
4655 }
4656
4657 static inline void
4658 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4659 {
4660         struct thread *t = &p->threads[p->thread_id];
4661         struct instruction *ip = t->ip;
4662
4663         /* Structs. */
4664         __instr_regwr_rmm_exec(p, t, ip);
4665
4666         /* Thread. */
4667         thread_ip_inc(p);
4668 }
4669
4670 static inline void
4671 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4672 {
4673         struct thread *t = &p->threads[p->thread_id];
4674         struct instruction *ip = t->ip;
4675
4676         /* Structs. */
4677         __instr_regwr_rhi_exec(p, t, ip);
4678
4679         /* Thread. */
4680         thread_ip_inc(p);
4681 }
4682
4683 static inline void
4684 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4685 {
4686         struct thread *t = &p->threads[p->thread_id];
4687         struct instruction *ip = t->ip;
4688
4689         /* Structs. */
4690         __instr_regwr_rmi_exec(p, t, ip);
4691
4692         /* Thread. */
4693         thread_ip_inc(p);
4694 }
4695
4696 static inline void
4697 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4698 {
4699         struct thread *t = &p->threads[p->thread_id];
4700         struct instruction *ip = t->ip;
4701
4702         /* Structs. */
4703         __instr_regwr_rih_exec(p, t, ip);
4704
4705         /* Thread. */
4706         thread_ip_inc(p);
4707 }
4708
4709 static inline void
4710 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4711 {
4712         struct thread *t = &p->threads[p->thread_id];
4713         struct instruction *ip = t->ip;
4714
4715         /* Structs. */
4716         __instr_regwr_rim_exec(p, t, ip);
4717
4718         /* Thread. */
4719         thread_ip_inc(p);
4720 }
4721
4722 static inline void
4723 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4724 {
4725         struct thread *t = &p->threads[p->thread_id];
4726         struct instruction *ip = t->ip;
4727
4728         /* Structs. */
4729         __instr_regwr_rii_exec(p, t, ip);
4730
4731         /* Thread. */
4732         thread_ip_inc(p);
4733 }
4734
4735 static inline void
4736 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4737 {
4738         struct thread *t = &p->threads[p->thread_id];
4739         struct instruction *ip = t->ip;
4740
4741         /* Structs. */
4742         __instr_regadd_rhh_exec(p, t, ip);
4743
4744         /* Thread. */
4745         thread_ip_inc(p);
4746 }
4747
4748 static inline void
4749 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4750 {
4751         struct thread *t = &p->threads[p->thread_id];
4752         struct instruction *ip = t->ip;
4753
4754         /* Structs. */
4755         __instr_regadd_rhm_exec(p, t, ip);
4756
4757         /* Thread. */
4758         thread_ip_inc(p);
4759 }
4760
4761 static inline void
4762 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4763 {
4764         struct thread *t = &p->threads[p->thread_id];
4765         struct instruction *ip = t->ip;
4766
4767         /* Structs. */
4768         __instr_regadd_rmh_exec(p, t, ip);
4769
4770         /* Thread. */
4771         thread_ip_inc(p);
4772 }
4773
4774 static inline void
4775 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4776 {
4777         struct thread *t = &p->threads[p->thread_id];
4778         struct instruction *ip = t->ip;
4779
4780         /* Structs. */
4781         __instr_regadd_rmm_exec(p, t, ip);
4782
4783         /* Thread. */
4784         thread_ip_inc(p);
4785 }
4786
4787 static inline void
4788 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4789 {
4790         struct thread *t = &p->threads[p->thread_id];
4791         struct instruction *ip = t->ip;
4792
4793         /* Structs. */
4794         __instr_regadd_rhi_exec(p, t, ip);
4795
4796         /* Thread. */
4797         thread_ip_inc(p);
4798 }
4799
4800 static inline void
4801 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4802 {
4803         struct thread *t = &p->threads[p->thread_id];
4804         struct instruction *ip = t->ip;
4805
4806         /* Structs. */
4807         __instr_regadd_rmi_exec(p, t, ip);
4808
4809         /* Thread. */
4810         thread_ip_inc(p);
4811 }
4812
4813 static inline void
4814 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4815 {
4816         struct thread *t = &p->threads[p->thread_id];
4817         struct instruction *ip = t->ip;
4818
4819         /* Structs. */
4820         __instr_regadd_rih_exec(p, t, ip);
4821
4822         /* Thread. */
4823         thread_ip_inc(p);
4824 }
4825
4826 static inline void
4827 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4828 {
4829         struct thread *t = &p->threads[p->thread_id];
4830         struct instruction *ip = t->ip;
4831
4832         /* Structs. */
4833         __instr_regadd_rim_exec(p, t, ip);
4834
4835         /* Thread. */
4836         thread_ip_inc(p);
4837 }
4838
4839 static inline void
4840 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4841 {
4842         struct thread *t = &p->threads[p->thread_id];
4843         struct instruction *ip = t->ip;
4844
4845         /* Structs. */
4846         __instr_regadd_rii_exec(p, t, ip);
4847
4848         /* Thread. */
4849         thread_ip_inc(p);
4850 }
4851
4852 /*
4853  * metarray.
4854  */
4855 static struct metarray *
4856 metarray_find(struct rte_swx_pipeline *p, const char *name);
4857
4858 static int
4859 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4860                             struct action *action,
4861                             char **tokens,
4862                             int n_tokens,
4863                             struct instruction *instr,
4864                             struct instruction_data *data __rte_unused)
4865 {
4866         char *metarray = tokens[1], *idx = tokens[2];
4867         struct metarray *m;
4868         struct field *fidx;
4869         uint32_t idx_struct_id, idx_val;
4870
4871         CHECK(n_tokens == 3, EINVAL);
4872
4873         m = metarray_find(p, metarray);
4874         CHECK(m, EINVAL);
4875
4876         /* METPREFETCH_H, METPREFETCH_M. */
4877         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4878         if (fidx) {
4879                 CHECK(!fidx->var_size, EINVAL);
4880
4881                 instr->type = INSTR_METPREFETCH_M;
4882                 if (idx[0] == 'h')
4883                         instr->type = INSTR_METPREFETCH_H;
4884
4885                 instr->meter.metarray_id = m->id;
4886                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4887                 instr->meter.idx.n_bits = fidx->n_bits;
4888                 instr->meter.idx.offset = fidx->offset / 8;
4889                 return 0;
4890         }
4891
4892         /* METPREFETCH_I. */
4893         idx_val = strtoul(idx, &idx, 0);
4894         CHECK(!idx[0], EINVAL);
4895
4896         instr->type = INSTR_METPREFETCH_I;
4897         instr->meter.metarray_id = m->id;
4898         instr->meter.idx_val = idx_val;
4899         return 0;
4900 }
4901
4902 static int
4903 instr_meter_translate(struct rte_swx_pipeline *p,
4904                       struct action *action,
4905                       char **tokens,
4906                       int n_tokens,
4907                       struct instruction *instr,
4908                       struct instruction_data *data __rte_unused)
4909 {
4910         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4911         char *color_in = tokens[4], *color_out = tokens[5];
4912         struct metarray *m;
4913         struct field *fidx, *flength, *fcin, *fcout;
4914         uint32_t idx_struct_id, length_struct_id;
4915         uint32_t color_in_struct_id, color_out_struct_id;
4916
4917         CHECK(n_tokens == 6, EINVAL);
4918
4919         m = metarray_find(p, metarray);
4920         CHECK(m, EINVAL);
4921
4922         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4923
4924         flength = struct_field_parse(p, action, length, &length_struct_id);
4925         CHECK(flength, EINVAL);
4926         CHECK(!flength->var_size, EINVAL);
4927
4928         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4929
4930         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4931         CHECK(fcout, EINVAL);
4932         CHECK(!fcout->var_size, EINVAL);
4933
4934         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4935         if (fidx && fcin) {
4936                 CHECK(!fidx->var_size, EINVAL);
4937                 CHECK(!fcin->var_size, EINVAL);
4938
4939                 instr->type = INSTR_METER_MMM;
4940                 if (idx[0] == 'h' && length[0] == 'h')
4941                         instr->type = INSTR_METER_HHM;
4942                 if (idx[0] == 'h' && length[0] != 'h')
4943                         instr->type = INSTR_METER_HMM;
4944                 if (idx[0] != 'h' && length[0] == 'h')
4945                         instr->type = INSTR_METER_MHM;
4946
4947                 instr->meter.metarray_id = m->id;
4948
4949                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4950                 instr->meter.idx.n_bits = fidx->n_bits;
4951                 instr->meter.idx.offset = fidx->offset / 8;
4952
4953                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4954                 instr->meter.length.n_bits = flength->n_bits;
4955                 instr->meter.length.offset = flength->offset / 8;
4956
4957                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4958                 instr->meter.color_in.n_bits = fcin->n_bits;
4959                 instr->meter.color_in.offset = fcin->offset / 8;
4960
4961                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4962                 instr->meter.color_out.n_bits = fcout->n_bits;
4963                 instr->meter.color_out.offset = fcout->offset / 8;
4964         }
4965
4966         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4967         if (fidx && !fcin) {
4968                 uint32_t color_in_val;
4969
4970                 CHECK(!fidx->var_size, EINVAL);
4971
4972                 color_in_val = strtoul(color_in, &color_in, 0);
4973                 CHECK(!color_in[0], EINVAL);
4974
4975                 instr->type = INSTR_METER_MMI;
4976                 if (idx[0] == 'h' && length[0] == 'h')
4977                         instr->type = INSTR_METER_HHI;
4978                 if (idx[0] == 'h' && length[0] != 'h')
4979                         instr->type = INSTR_METER_HMI;
4980                 if (idx[0] != 'h' && length[0] == 'h')
4981                         instr->type = INSTR_METER_MHI;
4982
4983                 instr->meter.metarray_id = m->id;
4984
4985                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4986                 instr->meter.idx.n_bits = fidx->n_bits;
4987                 instr->meter.idx.offset = fidx->offset / 8;
4988
4989                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4990                 instr->meter.length.n_bits = flength->n_bits;
4991                 instr->meter.length.offset = flength->offset / 8;
4992
4993                 instr->meter.color_in_val = color_in_val;
4994
4995                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4996                 instr->meter.color_out.n_bits = fcout->n_bits;
4997                 instr->meter.color_out.offset = fcout->offset / 8;
4998         }
4999
5000         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5001         if (!fidx && fcin) {
5002                 uint32_t idx_val;
5003
5004                 idx_val = strtoul(idx, &idx, 0);
5005                 CHECK(!idx[0], EINVAL);
5006
5007                 CHECK(!fcin->var_size, EINVAL);
5008
5009                 instr->type = INSTR_METER_IMM;
5010                 if (length[0] == 'h')
5011                         instr->type = INSTR_METER_IHM;
5012
5013                 instr->meter.metarray_id = m->id;
5014
5015                 instr->meter.idx_val = idx_val;
5016
5017                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5018                 instr->meter.length.n_bits = flength->n_bits;
5019                 instr->meter.length.offset = flength->offset / 8;
5020
5021                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5022                 instr->meter.color_in.n_bits = fcin->n_bits;
5023                 instr->meter.color_in.offset = fcin->offset / 8;
5024
5025                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5026                 instr->meter.color_out.n_bits = fcout->n_bits;
5027                 instr->meter.color_out.offset = fcout->offset / 8;
5028         }
5029
5030         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5031         if (!fidx && !fcin) {
5032                 uint32_t idx_val, color_in_val;
5033
5034                 idx_val = strtoul(idx, &idx, 0);
5035                 CHECK(!idx[0], EINVAL);
5036
5037                 color_in_val = strtoul(color_in, &color_in, 0);
5038                 CHECK(!color_in[0], EINVAL);
5039
5040                 instr->type = INSTR_METER_IMI;
5041                 if (length[0] == 'h')
5042                         instr->type = INSTR_METER_IHI;
5043
5044                 instr->meter.metarray_id = m->id;
5045
5046                 instr->meter.idx_val = idx_val;
5047
5048                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5049                 instr->meter.length.n_bits = flength->n_bits;
5050                 instr->meter.length.offset = flength->offset / 8;
5051
5052                 instr->meter.color_in_val = color_in_val;
5053
5054                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5055                 instr->meter.color_out.n_bits = fcout->n_bits;
5056                 instr->meter.color_out.offset = fcout->offset / 8;
5057         }
5058
5059         return 0;
5060 }
5061
5062 static inline void
5063 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5064 {
5065         struct thread *t = &p->threads[p->thread_id];
5066         struct instruction *ip = t->ip;
5067
5068         /* Structs. */
5069         __instr_metprefetch_h_exec(p, t, ip);
5070
5071         /* Thread. */
5072         thread_ip_inc(p);
5073 }
5074
5075 static inline void
5076 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5077 {
5078         struct thread *t = &p->threads[p->thread_id];
5079         struct instruction *ip = t->ip;
5080
5081         /* Structs. */
5082         __instr_metprefetch_m_exec(p, t, ip);
5083
5084         /* Thread. */
5085         thread_ip_inc(p);
5086 }
5087
5088 static inline void
5089 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5090 {
5091         struct thread *t = &p->threads[p->thread_id];
5092         struct instruction *ip = t->ip;
5093
5094         /* Structs. */
5095         __instr_metprefetch_i_exec(p, t, ip);
5096
5097         /* Thread. */
5098         thread_ip_inc(p);
5099 }
5100
5101 static inline void
5102 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5103 {
5104         struct thread *t = &p->threads[p->thread_id];
5105         struct instruction *ip = t->ip;
5106
5107         /* Structs. */
5108         __instr_meter_hhm_exec(p, t, ip);
5109
5110         /* Thread. */
5111         thread_ip_inc(p);
5112 }
5113
5114 static inline void
5115 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5116 {
5117         struct thread *t = &p->threads[p->thread_id];
5118         struct instruction *ip = t->ip;
5119
5120         /* Structs. */
5121         __instr_meter_hhi_exec(p, t, ip);
5122
5123         /* Thread. */
5124         thread_ip_inc(p);
5125 }
5126
5127 static inline void
5128 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5129 {
5130         struct thread *t = &p->threads[p->thread_id];
5131         struct instruction *ip = t->ip;
5132
5133         /* Structs. */
5134         __instr_meter_hmm_exec(p, t, ip);
5135
5136         /* Thread. */
5137         thread_ip_inc(p);
5138 }
5139
5140 static inline void
5141 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5142 {
5143         struct thread *t = &p->threads[p->thread_id];
5144         struct instruction *ip = t->ip;
5145
5146         /* Structs. */
5147         __instr_meter_hmi_exec(p, t, ip);
5148
5149         /* Thread. */
5150         thread_ip_inc(p);
5151 }
5152
5153 static inline void
5154 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5155 {
5156         struct thread *t = &p->threads[p->thread_id];
5157         struct instruction *ip = t->ip;
5158
5159         /* Structs. */
5160         __instr_meter_mhm_exec(p, t, ip);
5161
5162         /* Thread. */
5163         thread_ip_inc(p);
5164 }
5165
5166 static inline void
5167 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5168 {
5169         struct thread *t = &p->threads[p->thread_id];
5170         struct instruction *ip = t->ip;
5171
5172         /* Structs. */
5173         __instr_meter_mhi_exec(p, t, ip);
5174
5175         /* Thread. */
5176         thread_ip_inc(p);
5177 }
5178
5179 static inline void
5180 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5181 {
5182         struct thread *t = &p->threads[p->thread_id];
5183         struct instruction *ip = t->ip;
5184
5185         /* Structs. */
5186         __instr_meter_mmm_exec(p, t, ip);
5187
5188         /* Thread. */
5189         thread_ip_inc(p);
5190 }
5191
5192 static inline void
5193 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5194 {
5195         struct thread *t = &p->threads[p->thread_id];
5196         struct instruction *ip = t->ip;
5197
5198         /* Structs. */
5199         __instr_meter_mmi_exec(p, t, ip);
5200
5201         /* Thread. */
5202         thread_ip_inc(p);
5203 }
5204
5205 static inline void
5206 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5207 {
5208         struct thread *t = &p->threads[p->thread_id];
5209         struct instruction *ip = t->ip;
5210
5211         /* Structs. */
5212         __instr_meter_ihm_exec(p, t, ip);
5213
5214         /* Thread. */
5215         thread_ip_inc(p);
5216 }
5217
5218 static inline void
5219 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5220 {
5221         struct thread *t = &p->threads[p->thread_id];
5222         struct instruction *ip = t->ip;
5223
5224         /* Structs. */
5225         __instr_meter_ihi_exec(p, t, ip);
5226
5227         /* Thread. */
5228         thread_ip_inc(p);
5229 }
5230
5231 static inline void
5232 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5233 {
5234         struct thread *t = &p->threads[p->thread_id];
5235         struct instruction *ip = t->ip;
5236
5237         /* Structs. */
5238         __instr_meter_imm_exec(p, t, ip);
5239
5240         /* Thread. */
5241         thread_ip_inc(p);
5242 }
5243
5244 static inline void
5245 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5246 {
5247         struct thread *t = &p->threads[p->thread_id];
5248         struct instruction *ip = t->ip;
5249
5250         /* Structs. */
5251         __instr_meter_imi_exec(p, t, ip);
5252
5253         /* Thread. */
5254         thread_ip_inc(p);
5255 }
5256
5257 /*
5258  * jmp.
5259  */
5260 static int
5261 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5262                     struct action *action __rte_unused,
5263                     char **tokens,
5264                     int n_tokens,
5265                     struct instruction *instr,
5266                     struct instruction_data *data)
5267 {
5268         CHECK(n_tokens == 2, EINVAL);
5269
5270         strcpy(data->jmp_label, tokens[1]);
5271
5272         instr->type = INSTR_JMP;
5273         instr->jmp.ip = NULL; /* Resolved later. */
5274         return 0;
5275 }
5276
5277 static int
5278 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5279                           struct action *action __rte_unused,
5280                           char **tokens,
5281                           int n_tokens,
5282                           struct instruction *instr,
5283                           struct instruction_data *data)
5284 {
5285         struct header *h;
5286
5287         CHECK(n_tokens == 3, EINVAL);
5288
5289         strcpy(data->jmp_label, tokens[1]);
5290
5291         h = header_parse(p, tokens[2]);
5292         CHECK(h, EINVAL);
5293
5294         instr->type = INSTR_JMP_VALID;
5295         instr->jmp.ip = NULL; /* Resolved later. */
5296         instr->jmp.header_id = h->id;
5297         return 0;
5298 }
5299
5300 static int
5301 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5302                             struct action *action __rte_unused,
5303                             char **tokens,
5304                             int n_tokens,
5305                             struct instruction *instr,
5306                             struct instruction_data *data)
5307 {
5308         struct header *h;
5309
5310         CHECK(n_tokens == 3, EINVAL);
5311
5312         strcpy(data->jmp_label, tokens[1]);
5313
5314         h = header_parse(p, tokens[2]);
5315         CHECK(h, EINVAL);
5316
5317         instr->type = INSTR_JMP_INVALID;
5318         instr->jmp.ip = NULL; /* Resolved later. */
5319         instr->jmp.header_id = h->id;
5320         return 0;
5321 }
5322
5323 static int
5324 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5325                         struct action *action,
5326                         char **tokens,
5327                         int n_tokens,
5328                         struct instruction *instr,
5329                         struct instruction_data *data)
5330 {
5331         CHECK(!action, EINVAL);
5332         CHECK(n_tokens == 2, EINVAL);
5333
5334         strcpy(data->jmp_label, tokens[1]);
5335
5336         instr->type = INSTR_JMP_HIT;
5337         instr->jmp.ip = NULL; /* Resolved later. */
5338         return 0;
5339 }
5340
5341 static int
5342 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5343                          struct action *action,
5344                          char **tokens,
5345                          int n_tokens,
5346                          struct instruction *instr,
5347                          struct instruction_data *data)
5348 {
5349         CHECK(!action, EINVAL);
5350         CHECK(n_tokens == 2, EINVAL);
5351
5352         strcpy(data->jmp_label, tokens[1]);
5353
5354         instr->type = INSTR_JMP_MISS;
5355         instr->jmp.ip = NULL; /* Resolved later. */
5356         return 0;
5357 }
5358
5359 static int
5360 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5361                                struct action *action,
5362                                char **tokens,
5363                                int n_tokens,
5364                                struct instruction *instr,
5365                                struct instruction_data *data)
5366 {
5367         struct action *a;
5368
5369         CHECK(!action, EINVAL);
5370         CHECK(n_tokens == 3, EINVAL);
5371
5372         strcpy(data->jmp_label, tokens[1]);
5373
5374         a = action_find(p, tokens[2]);
5375         CHECK(a, EINVAL);
5376
5377         instr->type = INSTR_JMP_ACTION_HIT;
5378         instr->jmp.ip = NULL; /* Resolved later. */
5379         instr->jmp.action_id = a->id;
5380         return 0;
5381 }
5382
5383 static int
5384 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5385                                 struct action *action,
5386                                 char **tokens,
5387                                 int n_tokens,
5388                                 struct instruction *instr,
5389                                 struct instruction_data *data)
5390 {
5391         struct action *a;
5392
5393         CHECK(!action, EINVAL);
5394         CHECK(n_tokens == 3, EINVAL);
5395
5396         strcpy(data->jmp_label, tokens[1]);
5397
5398         a = action_find(p, tokens[2]);
5399         CHECK(a, EINVAL);
5400
5401         instr->type = INSTR_JMP_ACTION_MISS;
5402         instr->jmp.ip = NULL; /* Resolved later. */
5403         instr->jmp.action_id = a->id;
5404         return 0;
5405 }
5406
5407 static int
5408 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5409                        struct action *action,
5410                        char **tokens,
5411                        int n_tokens,
5412                        struct instruction *instr,
5413                        struct instruction_data *data)
5414 {
5415         char *a = tokens[2], *b = tokens[3];
5416         struct field *fa, *fb;
5417         uint64_t b_val;
5418         uint32_t a_struct_id, b_struct_id;
5419
5420         CHECK(n_tokens == 4, EINVAL);
5421
5422         strcpy(data->jmp_label, tokens[1]);
5423
5424         fa = struct_field_parse(p, action, a, &a_struct_id);
5425         CHECK(fa, EINVAL);
5426         CHECK(!fa->var_size, EINVAL);
5427
5428         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5429         fb = struct_field_parse(p, action, b, &b_struct_id);
5430         if (fb) {
5431                 CHECK(!fb->var_size, EINVAL);
5432
5433                 instr->type = INSTR_JMP_EQ;
5434                 if (a[0] != 'h' && b[0] == 'h')
5435                         instr->type = INSTR_JMP_EQ_MH;
5436                 if (a[0] == 'h' && b[0] != 'h')
5437                         instr->type = INSTR_JMP_EQ_HM;
5438                 if (a[0] == 'h' && b[0] == 'h')
5439                         instr->type = INSTR_JMP_EQ_HH;
5440                 instr->jmp.ip = NULL; /* Resolved later. */
5441
5442                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5443                 instr->jmp.a.n_bits = fa->n_bits;
5444                 instr->jmp.a.offset = fa->offset / 8;
5445                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5446                 instr->jmp.b.n_bits = fb->n_bits;
5447                 instr->jmp.b.offset = fb->offset / 8;
5448                 return 0;
5449         }
5450
5451         /* JMP_EQ_I. */
5452         b_val = strtoull(b, &b, 0);
5453         CHECK(!b[0], EINVAL);
5454
5455         if (a[0] == 'h')
5456                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5457
5458         instr->type = INSTR_JMP_EQ_I;
5459         instr->jmp.ip = NULL; /* Resolved later. */
5460         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5461         instr->jmp.a.n_bits = fa->n_bits;
5462         instr->jmp.a.offset = fa->offset / 8;
5463         instr->jmp.b_val = b_val;
5464         return 0;
5465 }
5466
5467 static int
5468 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5469                         struct action *action,
5470                         char **tokens,
5471                         int n_tokens,
5472                         struct instruction *instr,
5473                         struct instruction_data *data)
5474 {
5475         char *a = tokens[2], *b = tokens[3];
5476         struct field *fa, *fb;
5477         uint64_t b_val;
5478         uint32_t a_struct_id, b_struct_id;
5479
5480         CHECK(n_tokens == 4, EINVAL);
5481
5482         strcpy(data->jmp_label, tokens[1]);
5483
5484         fa = struct_field_parse(p, action, a, &a_struct_id);
5485         CHECK(fa, EINVAL);
5486         CHECK(!fa->var_size, EINVAL);
5487
5488         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5489         fb = struct_field_parse(p, action, b, &b_struct_id);
5490         if (fb) {
5491                 CHECK(!fb->var_size, EINVAL);
5492
5493                 instr->type = INSTR_JMP_NEQ;
5494                 if (a[0] != 'h' && b[0] == 'h')
5495                         instr->type = INSTR_JMP_NEQ_MH;
5496                 if (a[0] == 'h' && b[0] != 'h')
5497                         instr->type = INSTR_JMP_NEQ_HM;
5498                 if (a[0] == 'h' && b[0] == 'h')
5499                         instr->type = INSTR_JMP_NEQ_HH;
5500                 instr->jmp.ip = NULL; /* Resolved later. */
5501
5502                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5503                 instr->jmp.a.n_bits = fa->n_bits;
5504                 instr->jmp.a.offset = fa->offset / 8;
5505                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5506                 instr->jmp.b.n_bits = fb->n_bits;
5507                 instr->jmp.b.offset = fb->offset / 8;
5508                 return 0;
5509         }
5510
5511         /* JMP_NEQ_I. */
5512         b_val = strtoull(b, &b, 0);
5513         CHECK(!b[0], EINVAL);
5514
5515         if (a[0] == 'h')
5516                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5517
5518         instr->type = INSTR_JMP_NEQ_I;
5519         instr->jmp.ip = NULL; /* Resolved later. */
5520         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5521         instr->jmp.a.n_bits = fa->n_bits;
5522         instr->jmp.a.offset = fa->offset / 8;
5523         instr->jmp.b_val = b_val;
5524         return 0;
5525 }
5526
5527 static int
5528 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5529                        struct action *action,
5530                        char **tokens,
5531                        int n_tokens,
5532                        struct instruction *instr,
5533                        struct instruction_data *data)
5534 {
5535         char *a = tokens[2], *b = tokens[3];
5536         struct field *fa, *fb;
5537         uint64_t b_val;
5538         uint32_t a_struct_id, b_struct_id;
5539
5540         CHECK(n_tokens == 4, EINVAL);
5541
5542         strcpy(data->jmp_label, tokens[1]);
5543
5544         fa = struct_field_parse(p, action, a, &a_struct_id);
5545         CHECK(fa, EINVAL);
5546         CHECK(!fa->var_size, EINVAL);
5547
5548         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5549         fb = struct_field_parse(p, action, b, &b_struct_id);
5550         if (fb) {
5551                 CHECK(!fb->var_size, EINVAL);
5552
5553                 instr->type = INSTR_JMP_LT;
5554                 if (a[0] == 'h' && b[0] != 'h')
5555                         instr->type = INSTR_JMP_LT_HM;
5556                 if (a[0] != 'h' && b[0] == 'h')
5557                         instr->type = INSTR_JMP_LT_MH;
5558                 if (a[0] == 'h' && b[0] == 'h')
5559                         instr->type = INSTR_JMP_LT_HH;
5560                 instr->jmp.ip = NULL; /* Resolved later. */
5561
5562                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5563                 instr->jmp.a.n_bits = fa->n_bits;
5564                 instr->jmp.a.offset = fa->offset / 8;
5565                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5566                 instr->jmp.b.n_bits = fb->n_bits;
5567                 instr->jmp.b.offset = fb->offset / 8;
5568                 return 0;
5569         }
5570
5571         /* JMP_LT_MI, JMP_LT_HI. */
5572         b_val = strtoull(b, &b, 0);
5573         CHECK(!b[0], EINVAL);
5574
5575         instr->type = INSTR_JMP_LT_MI;
5576         if (a[0] == 'h')
5577                 instr->type = INSTR_JMP_LT_HI;
5578         instr->jmp.ip = NULL; /* Resolved later. */
5579
5580         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5581         instr->jmp.a.n_bits = fa->n_bits;
5582         instr->jmp.a.offset = fa->offset / 8;
5583         instr->jmp.b_val = b_val;
5584         return 0;
5585 }
5586
5587 static int
5588 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5589                        struct action *action,
5590                        char **tokens,
5591                        int n_tokens,
5592                        struct instruction *instr,
5593                        struct instruction_data *data)
5594 {
5595         char *a = tokens[2], *b = tokens[3];
5596         struct field *fa, *fb;
5597         uint64_t b_val;
5598         uint32_t a_struct_id, b_struct_id;
5599
5600         CHECK(n_tokens == 4, EINVAL);
5601
5602         strcpy(data->jmp_label, tokens[1]);
5603
5604         fa = struct_field_parse(p, action, a, &a_struct_id);
5605         CHECK(fa, EINVAL);
5606         CHECK(!fa->var_size, EINVAL);
5607
5608         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5609         fb = struct_field_parse(p, action, b, &b_struct_id);
5610         if (fb) {
5611                 CHECK(!fb->var_size, EINVAL);
5612
5613                 instr->type = INSTR_JMP_GT;
5614                 if (a[0] == 'h' && b[0] != 'h')
5615                         instr->type = INSTR_JMP_GT_HM;
5616                 if (a[0] != 'h' && b[0] == 'h')
5617                         instr->type = INSTR_JMP_GT_MH;
5618                 if (a[0] == 'h' && b[0] == 'h')
5619                         instr->type = INSTR_JMP_GT_HH;
5620                 instr->jmp.ip = NULL; /* Resolved later. */
5621
5622                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5623                 instr->jmp.a.n_bits = fa->n_bits;
5624                 instr->jmp.a.offset = fa->offset / 8;
5625                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5626                 instr->jmp.b.n_bits = fb->n_bits;
5627                 instr->jmp.b.offset = fb->offset / 8;
5628                 return 0;
5629         }
5630
5631         /* JMP_GT_MI, JMP_GT_HI. */
5632         b_val = strtoull(b, &b, 0);
5633         CHECK(!b[0], EINVAL);
5634
5635         instr->type = INSTR_JMP_GT_MI;
5636         if (a[0] == 'h')
5637                 instr->type = INSTR_JMP_GT_HI;
5638         instr->jmp.ip = NULL; /* Resolved later. */
5639
5640         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5641         instr->jmp.a.n_bits = fa->n_bits;
5642         instr->jmp.a.offset = fa->offset / 8;
5643         instr->jmp.b_val = b_val;
5644         return 0;
5645 }
5646
5647 static inline void
5648 instr_jmp_exec(struct rte_swx_pipeline *p)
5649 {
5650         struct thread *t = &p->threads[p->thread_id];
5651         struct instruction *ip = t->ip;
5652
5653         TRACE("[Thread %2u] jmp\n", p->thread_id);
5654
5655         thread_ip_set(t, ip->jmp.ip);
5656 }
5657
5658 static inline void
5659 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5660 {
5661         struct thread *t = &p->threads[p->thread_id];
5662         struct instruction *ip = t->ip;
5663         uint32_t header_id = ip->jmp.header_id;
5664
5665         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5666
5667         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5668 }
5669
5670 static inline void
5671 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5672 {
5673         struct thread *t = &p->threads[p->thread_id];
5674         struct instruction *ip = t->ip;
5675         uint32_t header_id = ip->jmp.header_id;
5676
5677         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5678
5679         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5680 }
5681
5682 static inline void
5683 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5684 {
5685         struct thread *t = &p->threads[p->thread_id];
5686         struct instruction *ip = t->ip;
5687         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5688
5689         TRACE("[Thread %2u] jmph\n", p->thread_id);
5690
5691         t->ip = ip_next[t->hit];
5692 }
5693
5694 static inline void
5695 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5696 {
5697         struct thread *t = &p->threads[p->thread_id];
5698         struct instruction *ip = t->ip;
5699         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5700
5701         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5702
5703         t->ip = ip_next[t->hit];
5704 }
5705
5706 static inline void
5707 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5708 {
5709         struct thread *t = &p->threads[p->thread_id];
5710         struct instruction *ip = t->ip;
5711
5712         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5713
5714         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5715 }
5716
5717 static inline void
5718 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5719 {
5720         struct thread *t = &p->threads[p->thread_id];
5721         struct instruction *ip = t->ip;
5722
5723         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5724
5725         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5726 }
5727
5728 static inline void
5729 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5730 {
5731         struct thread *t = &p->threads[p->thread_id];
5732         struct instruction *ip = t->ip;
5733
5734         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5735
5736         JMP_CMP(t, ip, ==);
5737 }
5738
5739 static inline void
5740 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5741 {
5742         struct thread *t = &p->threads[p->thread_id];
5743         struct instruction *ip = t->ip;
5744
5745         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5746
5747         JMP_CMP_MH(t, ip, ==);
5748 }
5749
5750 static inline void
5751 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5752 {
5753         struct thread *t = &p->threads[p->thread_id];
5754         struct instruction *ip = t->ip;
5755
5756         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5757
5758         JMP_CMP_HM(t, ip, ==);
5759 }
5760
5761 static inline void
5762 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5763 {
5764         struct thread *t = &p->threads[p->thread_id];
5765         struct instruction *ip = t->ip;
5766
5767         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5768
5769         JMP_CMP_HH_FAST(t, ip, ==);
5770 }
5771
5772 static inline void
5773 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5774 {
5775         struct thread *t = &p->threads[p->thread_id];
5776         struct instruction *ip = t->ip;
5777
5778         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5779
5780         JMP_CMP_I(t, ip, ==);
5781 }
5782
5783 static inline void
5784 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5785 {
5786         struct thread *t = &p->threads[p->thread_id];
5787         struct instruction *ip = t->ip;
5788
5789         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5790
5791         JMP_CMP(t, ip, !=);
5792 }
5793
5794 static inline void
5795 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5796 {
5797         struct thread *t = &p->threads[p->thread_id];
5798         struct instruction *ip = t->ip;
5799
5800         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5801
5802         JMP_CMP_MH(t, ip, !=);
5803 }
5804
5805 static inline void
5806 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5807 {
5808         struct thread *t = &p->threads[p->thread_id];
5809         struct instruction *ip = t->ip;
5810
5811         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5812
5813         JMP_CMP_HM(t, ip, !=);
5814 }
5815
5816 static inline void
5817 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5818 {
5819         struct thread *t = &p->threads[p->thread_id];
5820         struct instruction *ip = t->ip;
5821
5822         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5823
5824         JMP_CMP_HH_FAST(t, ip, !=);
5825 }
5826
5827 static inline void
5828 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5829 {
5830         struct thread *t = &p->threads[p->thread_id];
5831         struct instruction *ip = t->ip;
5832
5833         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5834
5835         JMP_CMP_I(t, ip, !=);
5836 }
5837
5838 static inline void
5839 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5840 {
5841         struct thread *t = &p->threads[p->thread_id];
5842         struct instruction *ip = t->ip;
5843
5844         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5845
5846         JMP_CMP(t, ip, <);
5847 }
5848
5849 static inline void
5850 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5851 {
5852         struct thread *t = &p->threads[p->thread_id];
5853         struct instruction *ip = t->ip;
5854
5855         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5856
5857         JMP_CMP_MH(t, ip, <);
5858 }
5859
5860 static inline void
5861 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5862 {
5863         struct thread *t = &p->threads[p->thread_id];
5864         struct instruction *ip = t->ip;
5865
5866         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5867
5868         JMP_CMP_HM(t, ip, <);
5869 }
5870
5871 static inline void
5872 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5873 {
5874         struct thread *t = &p->threads[p->thread_id];
5875         struct instruction *ip = t->ip;
5876
5877         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5878
5879         JMP_CMP_HH(t, ip, <);
5880 }
5881
5882 static inline void
5883 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5884 {
5885         struct thread *t = &p->threads[p->thread_id];
5886         struct instruction *ip = t->ip;
5887
5888         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5889
5890         JMP_CMP_MI(t, ip, <);
5891 }
5892
5893 static inline void
5894 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5895 {
5896         struct thread *t = &p->threads[p->thread_id];
5897         struct instruction *ip = t->ip;
5898
5899         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5900
5901         JMP_CMP_HI(t, ip, <);
5902 }
5903
5904 static inline void
5905 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5906 {
5907         struct thread *t = &p->threads[p->thread_id];
5908         struct instruction *ip = t->ip;
5909
5910         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5911
5912         JMP_CMP(t, ip, >);
5913 }
5914
5915 static inline void
5916 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5917 {
5918         struct thread *t = &p->threads[p->thread_id];
5919         struct instruction *ip = t->ip;
5920
5921         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5922
5923         JMP_CMP_MH(t, ip, >);
5924 }
5925
5926 static inline void
5927 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5928 {
5929         struct thread *t = &p->threads[p->thread_id];
5930         struct instruction *ip = t->ip;
5931
5932         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5933
5934         JMP_CMP_HM(t, ip, >);
5935 }
5936
5937 static inline void
5938 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5939 {
5940         struct thread *t = &p->threads[p->thread_id];
5941         struct instruction *ip = t->ip;
5942
5943         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5944
5945         JMP_CMP_HH(t, ip, >);
5946 }
5947
5948 static inline void
5949 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5950 {
5951         struct thread *t = &p->threads[p->thread_id];
5952         struct instruction *ip = t->ip;
5953
5954         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5955
5956         JMP_CMP_MI(t, ip, >);
5957 }
5958
5959 static inline void
5960 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5961 {
5962         struct thread *t = &p->threads[p->thread_id];
5963         struct instruction *ip = t->ip;
5964
5965         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5966
5967         JMP_CMP_HI(t, ip, >);
5968 }
5969
5970 /*
5971  * return.
5972  */
5973 static int
5974 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5975                        struct action *action,
5976                        char **tokens __rte_unused,
5977                        int n_tokens,
5978                        struct instruction *instr,
5979                        struct instruction_data *data __rte_unused)
5980 {
5981         CHECK(action, EINVAL);
5982         CHECK(n_tokens == 1, EINVAL);
5983
5984         instr->type = INSTR_RETURN;
5985         return 0;
5986 }
5987
5988 static inline void
5989 instr_return_exec(struct rte_swx_pipeline *p)
5990 {
5991         struct thread *t = &p->threads[p->thread_id];
5992
5993         TRACE("[Thread %2u] return\n", p->thread_id);
5994
5995         t->ip = t->ret;
5996 }
5997
5998 static int
5999 instr_translate(struct rte_swx_pipeline *p,
6000                 struct action *action,
6001                 char *string,
6002                 struct instruction *instr,
6003                 struct instruction_data *data)
6004 {
6005         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6006         int n_tokens = 0, tpos = 0;
6007
6008         /* Parse the instruction string into tokens. */
6009         for ( ; ; ) {
6010                 char *token;
6011
6012                 token = strtok_r(string, " \t\v", &string);
6013                 if (!token)
6014                         break;
6015
6016                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6017                 CHECK_NAME(token, EINVAL);
6018
6019                 tokens[n_tokens] = token;
6020                 n_tokens++;
6021         }
6022
6023         CHECK(n_tokens, EINVAL);
6024
6025         /* Handle the optional instruction label. */
6026         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6027                 strcpy(data->label, tokens[0]);
6028
6029                 tpos += 2;
6030                 CHECK(n_tokens - tpos, EINVAL);
6031         }
6032
6033         /* Identify the instruction type. */
6034         if (!strcmp(tokens[tpos], "rx"))
6035                 return instr_rx_translate(p,
6036                                           action,
6037                                           &tokens[tpos],
6038                                           n_tokens - tpos,
6039                                           instr,
6040                                           data);
6041
6042         if (!strcmp(tokens[tpos], "tx"))
6043                 return instr_tx_translate(p,
6044                                           action,
6045                                           &tokens[tpos],
6046                                           n_tokens - tpos,
6047                                           instr,
6048                                           data);
6049
6050         if (!strcmp(tokens[tpos], "drop"))
6051                 return instr_drop_translate(p,
6052                                             action,
6053                                             &tokens[tpos],
6054                                             n_tokens - tpos,
6055                                             instr,
6056                                             data);
6057
6058         if (!strcmp(tokens[tpos], "mirror"))
6059                 return instr_mirror_translate(p,
6060                                               action,
6061                                               &tokens[tpos],
6062                                               n_tokens - tpos,
6063                                               instr,
6064                                               data);
6065
6066         if (!strcmp(tokens[tpos], "recirculate"))
6067                 return instr_recirculate_translate(p,
6068                                               action,
6069                                               &tokens[tpos],
6070                                               n_tokens - tpos,
6071                                               instr,
6072                                               data);
6073
6074         if (!strcmp(tokens[tpos], "recircid"))
6075                 return instr_recircid_translate(p,
6076                                               action,
6077                                               &tokens[tpos],
6078                                               n_tokens - tpos,
6079                                               instr,
6080                                               data);
6081
6082         if (!strcmp(tokens[tpos], "extract"))
6083                 return instr_hdr_extract_translate(p,
6084                                                    action,
6085                                                    &tokens[tpos],
6086                                                    n_tokens - tpos,
6087                                                    instr,
6088                                                    data);
6089
6090         if (!strcmp(tokens[tpos], "lookahead"))
6091                 return instr_hdr_lookahead_translate(p,
6092                                                      action,
6093                                                      &tokens[tpos],
6094                                                      n_tokens - tpos,
6095                                                      instr,
6096                                                      data);
6097
6098         if (!strcmp(tokens[tpos], "emit"))
6099                 return instr_hdr_emit_translate(p,
6100                                                 action,
6101                                                 &tokens[tpos],
6102                                                 n_tokens - tpos,
6103                                                 instr,
6104                                                 data);
6105
6106         if (!strcmp(tokens[tpos], "validate"))
6107                 return instr_hdr_validate_translate(p,
6108                                                     action,
6109                                                     &tokens[tpos],
6110                                                     n_tokens - tpos,
6111                                                     instr,
6112                                                     data);
6113
6114         if (!strcmp(tokens[tpos], "invalidate"))
6115                 return instr_hdr_invalidate_translate(p,
6116                                                       action,
6117                                                       &tokens[tpos],
6118                                                       n_tokens - tpos,
6119                                                       instr,
6120                                                       data);
6121
6122         if (!strcmp(tokens[tpos], "mov"))
6123                 return instr_mov_translate(p,
6124                                            action,
6125                                            &tokens[tpos],
6126                                            n_tokens - tpos,
6127                                            instr,
6128                                            data);
6129
6130         if (!strcmp(tokens[tpos], "add"))
6131                 return instr_alu_add_translate(p,
6132                                                action,
6133                                                &tokens[tpos],
6134                                                n_tokens - tpos,
6135                                                instr,
6136                                                data);
6137
6138         if (!strcmp(tokens[tpos], "sub"))
6139                 return instr_alu_sub_translate(p,
6140                                                action,
6141                                                &tokens[tpos],
6142                                                n_tokens - tpos,
6143                                                instr,
6144                                                data);
6145
6146         if (!strcmp(tokens[tpos], "ckadd"))
6147                 return instr_alu_ckadd_translate(p,
6148                                                  action,
6149                                                  &tokens[tpos],
6150                                                  n_tokens - tpos,
6151                                                  instr,
6152                                                  data);
6153
6154         if (!strcmp(tokens[tpos], "cksub"))
6155                 return instr_alu_cksub_translate(p,
6156                                                  action,
6157                                                  &tokens[tpos],
6158                                                  n_tokens - tpos,
6159                                                  instr,
6160                                                  data);
6161
6162         if (!strcmp(tokens[tpos], "and"))
6163                 return instr_alu_and_translate(p,
6164                                                action,
6165                                                &tokens[tpos],
6166                                                n_tokens - tpos,
6167                                                instr,
6168                                                data);
6169
6170         if (!strcmp(tokens[tpos], "or"))
6171                 return instr_alu_or_translate(p,
6172                                               action,
6173                                               &tokens[tpos],
6174                                               n_tokens - tpos,
6175                                               instr,
6176                                               data);
6177
6178         if (!strcmp(tokens[tpos], "xor"))
6179                 return instr_alu_xor_translate(p,
6180                                                action,
6181                                                &tokens[tpos],
6182                                                n_tokens - tpos,
6183                                                instr,
6184                                                data);
6185
6186         if (!strcmp(tokens[tpos], "shl"))
6187                 return instr_alu_shl_translate(p,
6188                                                action,
6189                                                &tokens[tpos],
6190                                                n_tokens - tpos,
6191                                                instr,
6192                                                data);
6193
6194         if (!strcmp(tokens[tpos], "shr"))
6195                 return instr_alu_shr_translate(p,
6196                                                action,
6197                                                &tokens[tpos],
6198                                                n_tokens - tpos,
6199                                                instr,
6200                                                data);
6201
6202         if (!strcmp(tokens[tpos], "regprefetch"))
6203                 return instr_regprefetch_translate(p,
6204                                                    action,
6205                                                    &tokens[tpos],
6206                                                    n_tokens - tpos,
6207                                                    instr,
6208                                                    data);
6209
6210         if (!strcmp(tokens[tpos], "regrd"))
6211                 return instr_regrd_translate(p,
6212                                              action,
6213                                              &tokens[tpos],
6214                                              n_tokens - tpos,
6215                                              instr,
6216                                              data);
6217
6218         if (!strcmp(tokens[tpos], "regwr"))
6219                 return instr_regwr_translate(p,
6220                                              action,
6221                                              &tokens[tpos],
6222                                              n_tokens - tpos,
6223                                              instr,
6224                                              data);
6225
6226         if (!strcmp(tokens[tpos], "regadd"))
6227                 return instr_regadd_translate(p,
6228                                               action,
6229                                               &tokens[tpos],
6230                                               n_tokens - tpos,
6231                                               instr,
6232                                               data);
6233
6234         if (!strcmp(tokens[tpos], "metprefetch"))
6235                 return instr_metprefetch_translate(p,
6236                                                    action,
6237                                                    &tokens[tpos],
6238                                                    n_tokens - tpos,
6239                                                    instr,
6240                                                    data);
6241
6242         if (!strcmp(tokens[tpos], "meter"))
6243                 return instr_meter_translate(p,
6244                                              action,
6245                                              &tokens[tpos],
6246                                              n_tokens - tpos,
6247                                              instr,
6248                                              data);
6249
6250         if (!strcmp(tokens[tpos], "table"))
6251                 return instr_table_translate(p,
6252                                              action,
6253                                              &tokens[tpos],
6254                                              n_tokens - tpos,
6255                                              instr,
6256                                              data);
6257
6258         if (!strcmp(tokens[tpos], "learn"))
6259                 return instr_learn_translate(p,
6260                                              action,
6261                                              &tokens[tpos],
6262                                              n_tokens - tpos,
6263                                              instr,
6264                                              data);
6265         if (!strcmp(tokens[tpos], "rearm"))
6266                 return instr_rearm_translate(p,
6267                                              action,
6268                                              &tokens[tpos],
6269                                              n_tokens - tpos,
6270                                              instr,
6271                                              data);
6272
6273         if (!strcmp(tokens[tpos], "forget"))
6274                 return instr_forget_translate(p,
6275                                               action,
6276                                               &tokens[tpos],
6277                                               n_tokens - tpos,
6278                                               instr,
6279                                               data);
6280
6281         if (!strcmp(tokens[tpos], "extern"))
6282                 return instr_extern_translate(p,
6283                                               action,
6284                                               &tokens[tpos],
6285                                               n_tokens - tpos,
6286                                               instr,
6287                                               data);
6288
6289         if (!strcmp(tokens[tpos], "hash"))
6290                 return instr_hash_translate(p,
6291                                             action,
6292                                             &tokens[tpos],
6293                                             n_tokens - tpos,
6294                                             instr,
6295                                             data);
6296
6297         if (!strcmp(tokens[tpos], "jmp"))
6298                 return instr_jmp_translate(p,
6299                                            action,
6300                                            &tokens[tpos],
6301                                            n_tokens - tpos,
6302                                            instr,
6303                                            data);
6304
6305         if (!strcmp(tokens[tpos], "jmpv"))
6306                 return instr_jmp_valid_translate(p,
6307                                                  action,
6308                                                  &tokens[tpos],
6309                                                  n_tokens - tpos,
6310                                                  instr,
6311                                                  data);
6312
6313         if (!strcmp(tokens[tpos], "jmpnv"))
6314                 return instr_jmp_invalid_translate(p,
6315                                                    action,
6316                                                    &tokens[tpos],
6317                                                    n_tokens - tpos,
6318                                                    instr,
6319                                                    data);
6320
6321         if (!strcmp(tokens[tpos], "jmph"))
6322                 return instr_jmp_hit_translate(p,
6323                                                action,
6324                                                &tokens[tpos],
6325                                                n_tokens - tpos,
6326                                                instr,
6327                                                data);
6328
6329         if (!strcmp(tokens[tpos], "jmpnh"))
6330                 return instr_jmp_miss_translate(p,
6331                                                 action,
6332                                                 &tokens[tpos],
6333                                                 n_tokens - tpos,
6334                                                 instr,
6335                                                 data);
6336
6337         if (!strcmp(tokens[tpos], "jmpa"))
6338                 return instr_jmp_action_hit_translate(p,
6339                                                       action,
6340                                                       &tokens[tpos],
6341                                                       n_tokens - tpos,
6342                                                       instr,
6343                                                       data);
6344
6345         if (!strcmp(tokens[tpos], "jmpna"))
6346                 return instr_jmp_action_miss_translate(p,
6347                                                        action,
6348                                                        &tokens[tpos],
6349                                                        n_tokens - tpos,
6350                                                        instr,
6351                                                        data);
6352
6353         if (!strcmp(tokens[tpos], "jmpeq"))
6354                 return instr_jmp_eq_translate(p,
6355                                               action,
6356                                               &tokens[tpos],
6357                                               n_tokens - tpos,
6358                                               instr,
6359                                               data);
6360
6361         if (!strcmp(tokens[tpos], "jmpneq"))
6362                 return instr_jmp_neq_translate(p,
6363                                                action,
6364                                                &tokens[tpos],
6365                                                n_tokens - tpos,
6366                                                instr,
6367                                                data);
6368
6369         if (!strcmp(tokens[tpos], "jmplt"))
6370                 return instr_jmp_lt_translate(p,
6371                                               action,
6372                                               &tokens[tpos],
6373                                               n_tokens - tpos,
6374                                               instr,
6375                                               data);
6376
6377         if (!strcmp(tokens[tpos], "jmpgt"))
6378                 return instr_jmp_gt_translate(p,
6379                                               action,
6380                                               &tokens[tpos],
6381                                               n_tokens - tpos,
6382                                               instr,
6383                                               data);
6384
6385         if (!strcmp(tokens[tpos], "return"))
6386                 return instr_return_translate(p,
6387                                               action,
6388                                               &tokens[tpos],
6389                                               n_tokens - tpos,
6390                                               instr,
6391                                               data);
6392
6393         return -EINVAL;
6394 }
6395
6396 static struct instruction_data *
6397 label_find(struct instruction_data *data, uint32_t n, const char *label)
6398 {
6399         uint32_t i;
6400
6401         for (i = 0; i < n; i++)
6402                 if (!strcmp(label, data[i].label))
6403                         return &data[i];
6404
6405         return NULL;
6406 }
6407
6408 static uint32_t
6409 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6410 {
6411         uint32_t count = 0, i;
6412
6413         if (!label[0])
6414                 return 0;
6415
6416         for (i = 0; i < n; i++)
6417                 if (!strcmp(label, data[i].jmp_label))
6418                         count++;
6419
6420         return count;
6421 }
6422
6423 static int
6424 instr_label_check(struct instruction_data *instruction_data,
6425                   uint32_t n_instructions)
6426 {
6427         uint32_t i;
6428
6429         /* Check that all instruction labels are unique. */
6430         for (i = 0; i < n_instructions; i++) {
6431                 struct instruction_data *data = &instruction_data[i];
6432                 char *label = data->label;
6433                 uint32_t j;
6434
6435                 if (!label[0])
6436                         continue;
6437
6438                 for (j = i + 1; j < n_instructions; j++)
6439                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6440         }
6441
6442         /* Check that no jump instruction (either conditional or not) can jump to itself (loop). */
6443         for (i = 0; i < n_instructions; i++) {
6444                 struct instruction_data *data = &instruction_data[i];
6445                 char *label = data->label;
6446                 char *jmp_label = data->jmp_label;
6447
6448                 /* Continue if this instruction does not have a label or it is not a jump. */
6449                 if (!label[0] || !jmp_label[0])
6450                         continue;
6451
6452                 CHECK(strcmp(label, jmp_label), EINVAL);
6453         }
6454
6455         /* Get users for each instruction label. */
6456         for (i = 0; i < n_instructions; i++) {
6457                 struct instruction_data *data = &instruction_data[i];
6458                 char *label = data->label;
6459
6460                 data->n_users = label_is_used(instruction_data,
6461                                               n_instructions,
6462                                               label);
6463         }
6464
6465         return 0;
6466 }
6467
6468 static int
6469 instr_jmp_resolve(struct instruction *instructions,
6470                   struct instruction_data *instruction_data,
6471                   uint32_t n_instructions)
6472 {
6473         uint32_t i;
6474
6475         for (i = 0; i < n_instructions; i++) {
6476                 struct instruction *instr = &instructions[i];
6477                 struct instruction_data *data = &instruction_data[i];
6478                 struct instruction_data *found;
6479
6480                 if (!instruction_is_jmp(instr))
6481                         continue;
6482
6483                 found = label_find(instruction_data,
6484                                    n_instructions,
6485                                    data->jmp_label);
6486                 CHECK(found, EINVAL);
6487
6488                 instr->jmp.ip = &instructions[found - instruction_data];
6489         }
6490
6491         return 0;
6492 }
6493
6494 static int
6495 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6496              struct action *a,
6497              struct instruction *instr,
6498              struct instruction_data *data __rte_unused,
6499              uint32_t n_instructions)
6500 {
6501         if (!a) {
6502                 enum instruction_type type;
6503                 uint32_t i;
6504
6505                 /* Check that the first instruction is rx. */
6506                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6507
6508                 /* Check that there is at least one tx instruction. */
6509                 for (i = 0; i < n_instructions; i++) {
6510                         type = instr[i].type;
6511
6512                         if (instruction_is_tx(type))
6513                                 break;
6514                 }
6515                 CHECK(i < n_instructions, EINVAL);
6516
6517                 /* Check that the last instruction is either tx or unconditional
6518                  * jump.
6519                  */
6520                 type = instr[n_instructions - 1].type;
6521                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6522         }
6523
6524         if (a) {
6525                 enum instruction_type type;
6526                 uint32_t i;
6527
6528                 /* Check that there is at least one return or tx instruction. */
6529                 for (i = 0; i < n_instructions; i++) {
6530                         type = instr[i].type;
6531
6532                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6533                                 break;
6534                 }
6535                 CHECK(i < n_instructions, EINVAL);
6536         }
6537
6538         return 0;
6539 }
6540
6541 static uint32_t
6542 instr_compact(struct instruction *instructions,
6543               struct instruction_data *instruction_data,
6544               uint32_t n_instructions)
6545 {
6546         uint32_t i, pos = 0;
6547
6548         /* Eliminate the invalid instructions that have been optimized out. */
6549         for (i = 0; i < n_instructions; i++) {
6550                 struct instruction *instr = &instructions[i];
6551                 struct instruction_data *data = &instruction_data[i];
6552
6553                 if (data->invalid)
6554                         continue;
6555
6556                 if (i != pos) {
6557                         memcpy(&instructions[pos], instr, sizeof(*instr));
6558                         memcpy(&instruction_data[pos], data, sizeof(*data));
6559                 }
6560
6561                 pos++;
6562         }
6563
6564         return pos;
6565 }
6566
6567 static int
6568 instr_pattern_extract_many_search(struct instruction *instr,
6569                                   struct instruction_data *data,
6570                                   uint32_t n_instr,
6571                                   uint32_t *n_pattern_instr)
6572 {
6573         uint32_t i;
6574
6575         for (i = 0; i < n_instr; i++) {
6576                 if (data[i].invalid)
6577                         break;
6578
6579                 if (instr[i].type != INSTR_HDR_EXTRACT)
6580                         break;
6581
6582                 if (i == RTE_DIM(instr->io.hdr.header_id))
6583                         break;
6584
6585                 if (i && data[i].n_users)
6586                         break;
6587         }
6588
6589         if (i < 2)
6590                 return 0;
6591
6592         *n_pattern_instr = i;
6593         return 1;
6594 }
6595
6596 static void
6597 instr_pattern_extract_many_replace(struct instruction *instr,
6598                                    struct instruction_data *data,
6599                                    uint32_t n_instr)
6600 {
6601         uint32_t i;
6602
6603         for (i = 1; i < n_instr; i++) {
6604                 instr[0].type++;
6605                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6606                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6607                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6608
6609                 data[i].invalid = 1;
6610         }
6611 }
6612
6613 static uint32_t
6614 instr_pattern_extract_many_optimize(struct instruction *instructions,
6615                                     struct instruction_data *instruction_data,
6616                                     uint32_t n_instructions)
6617 {
6618         uint32_t i;
6619
6620         for (i = 0; i < n_instructions; ) {
6621                 struct instruction *instr = &instructions[i];
6622                 struct instruction_data *data = &instruction_data[i];
6623                 uint32_t n_instr = 0;
6624                 int detected;
6625
6626                 /* Extract many. */
6627                 detected = instr_pattern_extract_many_search(instr,
6628                                                              data,
6629                                                              n_instructions - i,
6630                                                              &n_instr);
6631                 if (detected) {
6632                         instr_pattern_extract_many_replace(instr,
6633                                                            data,
6634                                                            n_instr);
6635                         i += n_instr;
6636                         continue;
6637                 }
6638
6639                 /* No pattern starting at the current instruction. */
6640                 i++;
6641         }
6642
6643         /* Eliminate the invalid instructions that have been optimized out. */
6644         n_instructions = instr_compact(instructions,
6645                                        instruction_data,
6646                                        n_instructions);
6647
6648         return n_instructions;
6649 }
6650
6651 static int
6652 instr_pattern_emit_many_tx_search(struct instruction *instr,
6653                                   struct instruction_data *data,
6654                                   uint32_t n_instr,
6655                                   uint32_t *n_pattern_instr)
6656 {
6657         uint32_t i;
6658
6659         for (i = 0; i < n_instr; i++) {
6660                 if (data[i].invalid)
6661                         break;
6662
6663                 if (instr[i].type != INSTR_HDR_EMIT)
6664                         break;
6665
6666                 if (i == RTE_DIM(instr->io.hdr.header_id))
6667                         break;
6668
6669                 if (i && data[i].n_users)
6670                         break;
6671         }
6672
6673         if (!i)
6674                 return 0;
6675
6676         if (instr[i].type != INSTR_TX)
6677                 return 0;
6678
6679         if (data[i].n_users)
6680                 return 0;
6681
6682         i++;
6683
6684         *n_pattern_instr = i;
6685         return 1;
6686 }
6687
6688 static void
6689 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6690                                    struct instruction_data *data,
6691                                    uint32_t n_instr)
6692 {
6693         uint32_t i;
6694
6695         /* Any emit instruction in addition to the first one. */
6696         for (i = 1; i < n_instr - 1; i++) {
6697                 instr[0].type++;
6698                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6699                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6700                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6701
6702                 data[i].invalid = 1;
6703         }
6704
6705         /* The TX instruction is the last one in the pattern. */
6706         instr[0].type++;
6707         instr[0].io.io.offset = instr[i].io.io.offset;
6708         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6709         data[i].invalid = 1;
6710 }
6711
6712 static uint32_t
6713 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6714                                     struct instruction_data *instruction_data,
6715                                     uint32_t n_instructions)
6716 {
6717         uint32_t i;
6718
6719         for (i = 0; i < n_instructions; ) {
6720                 struct instruction *instr = &instructions[i];
6721                 struct instruction_data *data = &instruction_data[i];
6722                 uint32_t n_instr = 0;
6723                 int detected;
6724
6725                 /* Emit many + TX. */
6726                 detected = instr_pattern_emit_many_tx_search(instr,
6727                                                              data,
6728                                                              n_instructions - i,
6729                                                              &n_instr);
6730                 if (detected) {
6731                         instr_pattern_emit_many_tx_replace(instr,
6732                                                            data,
6733                                                            n_instr);
6734                         i += n_instr;
6735                         continue;
6736                 }
6737
6738                 /* No pattern starting at the current instruction. */
6739                 i++;
6740         }
6741
6742         /* Eliminate the invalid instructions that have been optimized out. */
6743         n_instructions = instr_compact(instructions,
6744                                        instruction_data,
6745                                        n_instructions);
6746
6747         return n_instructions;
6748 }
6749
6750 static uint32_t
6751 action_arg_src_mov_count(struct action *a,
6752                          uint32_t arg_id,
6753                          struct instruction *instructions,
6754                          struct instruction_data *instruction_data,
6755                          uint32_t n_instructions);
6756
6757 static int
6758 instr_pattern_validate_mov_all_search(struct rte_swx_pipeline *p,
6759                                       struct action *a,
6760                                       struct instruction *instr,
6761                                       struct instruction_data *data,
6762                                       uint32_t n_instr,
6763                                       struct instruction *instructions,
6764                                       struct instruction_data *instruction_data,
6765                                       uint32_t n_instructions,
6766                                       uint32_t *n_pattern_instr)
6767 {
6768         struct header *h;
6769         uint32_t src_field_id, i, j;
6770
6771         /* Prerequisites. */
6772         if (!a || !a->st)
6773                 return 0;
6774
6775         /* First instruction: HDR_VALIDATE. Second instruction: MOV_HM. */
6776         if (data[0].invalid ||
6777             (instr[0].type != INSTR_HDR_VALIDATE) ||
6778             (n_instr < 2) ||
6779             data[1].invalid ||
6780             (instr[1].type != INSTR_MOV_HM) ||
6781             instr[1].mov.src.struct_id)
6782                 return 0;
6783
6784         h = header_find_by_struct_id(p, instr[0].valid.struct_id);
6785         if (!h ||
6786             h->st->var_size ||
6787             (n_instr < 1 + h->st->n_fields))
6788                 return 0;
6789
6790         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6791                 if (instr[1].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6792                         break;
6793
6794         if (src_field_id + h->st->n_fields > a->st->n_fields)
6795                 return 0;
6796
6797         /* Second and subsequent instructions: MOV_HM. */
6798         for (i = 0; i < h->st->n_fields; i++)
6799                 if (data[1 + i].invalid ||
6800                     data[1 + i].n_users ||
6801                     (instr[1 + i].type != INSTR_MOV_HM) ||
6802                     (instr[1 + i].mov.dst.struct_id != h->struct_id) ||
6803                     (instr[1 + i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6804                     (instr[1 + i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6805                     instr[1 + i].mov.src.struct_id ||
6806                     (instr[1 + i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6807                     (instr[1 + i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6808                     (instr[1 + i].mov.dst.n_bits != instr[1 + i].mov.src.n_bits))
6809                         return 0;
6810
6811         /* Check that none of the action args that are used as source for this
6812          * DMA transfer are not used as source in any other mov instruction.
6813          */
6814         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6815                 uint32_t n_users;
6816
6817                 n_users = action_arg_src_mov_count(a,
6818                                                    j,
6819                                                    instructions,
6820                                                    instruction_data,
6821                                                    n_instructions);
6822                 if (n_users > 1)
6823                         return 0;
6824         }
6825
6826         *n_pattern_instr = 1 + h->st->n_fields;
6827         return 1;
6828 }
6829
6830 static void
6831 instr_pattern_validate_mov_all_replace(struct rte_swx_pipeline *p,
6832                                        struct action *a,
6833                                        struct instruction *instr,
6834                                        struct instruction_data *data,
6835                                        uint32_t n_instr)
6836 {
6837         struct header *h;
6838         uint32_t src_field_id, src_offset, i;
6839
6840         /* Read from the instructions before they are modified. */
6841         h = header_find_by_struct_id(p, instr[1].mov.dst.struct_id);
6842         if (!h)
6843                 return;
6844
6845         src_offset = instr[1].mov.src.offset;
6846
6847         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6848                 if (src_offset == a->st->fields[src_field_id].offset / 8)
6849                         break;
6850
6851         /* Modify the instructions. */
6852         instr[0].type = INSTR_DMA_HT;
6853         instr[0].dma.dst.header_id[0] = h->id;
6854         instr[0].dma.dst.struct_id[0] = h->struct_id;
6855         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6856         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6857
6858         for (i = 1; i < n_instr; i++)
6859                 data[i].invalid = 1;
6860
6861         /* Update the endianness of the action arguments to header endianness. */
6862         for (i = 0; i < h->st->n_fields; i++)
6863                 a->args_endianness[src_field_id + i] = 1;
6864 }
6865
6866 static uint32_t
6867 instr_pattern_validate_mov_all_optimize(struct rte_swx_pipeline *p,
6868                                         struct action *a,
6869                                         struct instruction *instructions,
6870                                         struct instruction_data *instruction_data,
6871                                         uint32_t n_instructions)
6872 {
6873         uint32_t i;
6874
6875         if (!a || !a->st)
6876                 return n_instructions;
6877
6878         for (i = 0; i < n_instructions; ) {
6879                 struct instruction *instr = &instructions[i];
6880                 struct instruction_data *data = &instruction_data[i];
6881                 uint32_t n_instr = 0;
6882                 int detected;
6883
6884                 /* Validate + mov all. */
6885                 detected = instr_pattern_validate_mov_all_search(p,
6886                                                                  a,
6887                                                                  instr,
6888                                                                  data,
6889                                                                  n_instructions - i,
6890                                                                  instructions,
6891                                                                  instruction_data,
6892                                                                  n_instructions,
6893                                                                  &n_instr);
6894                 if (detected) {
6895                         instr_pattern_validate_mov_all_replace(p, a, instr, data, n_instr);
6896                         i += n_instr;
6897                         continue;
6898                 }
6899
6900                 /* No pattern starting at the current instruction. */
6901                 i++;
6902         }
6903
6904         /* Eliminate the invalid instructions that have been optimized out. */
6905         n_instructions = instr_compact(instructions,
6906                                        instruction_data,
6907                                        n_instructions);
6908
6909         return n_instructions;
6910 }
6911
6912 static int
6913 instr_pattern_dma_many_search(struct instruction *instr,
6914                               struct instruction_data *data,
6915                               uint32_t n_instr,
6916                               uint32_t *n_pattern_instr)
6917 {
6918         uint32_t i;
6919
6920         for (i = 0; i < n_instr; i++) {
6921                 if (data[i].invalid)
6922                         break;
6923
6924                 if (instr[i].type != INSTR_DMA_HT)
6925                         break;
6926
6927                 if (i == RTE_DIM(instr->dma.dst.header_id))
6928                         break;
6929
6930                 if (i && data[i].n_users)
6931                         break;
6932         }
6933
6934         if (i < 2)
6935                 return 0;
6936
6937         *n_pattern_instr = i;
6938         return 1;
6939 }
6940
6941 static void
6942 instr_pattern_dma_many_replace(struct instruction *instr,
6943                                struct instruction_data *data,
6944                                uint32_t n_instr)
6945 {
6946         uint32_t i;
6947
6948         for (i = 1; i < n_instr; i++) {
6949                 instr[0].type++;
6950                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6951                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6952                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6953                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6954
6955                 data[i].invalid = 1;
6956         }
6957 }
6958
6959 static uint32_t
6960 instr_pattern_dma_many_optimize(struct instruction *instructions,
6961                struct instruction_data *instruction_data,
6962                uint32_t n_instructions)
6963 {
6964         uint32_t i;
6965
6966         for (i = 0; i < n_instructions; ) {
6967                 struct instruction *instr = &instructions[i];
6968                 struct instruction_data *data = &instruction_data[i];
6969                 uint32_t n_instr = 0;
6970                 int detected;
6971
6972                 /* DMA many. */
6973                 detected = instr_pattern_dma_many_search(instr,
6974                                                          data,
6975                                                          n_instructions - i,
6976                                                          &n_instr);
6977                 if (detected) {
6978                         instr_pattern_dma_many_replace(instr, data, n_instr);
6979                         i += n_instr;
6980                         continue;
6981                 }
6982
6983                 /* No pattern starting at the current instruction. */
6984                 i++;
6985         }
6986
6987         /* Eliminate the invalid instructions that have been optimized out. */
6988         n_instructions = instr_compact(instructions,
6989                                        instruction_data,
6990                                        n_instructions);
6991
6992         return n_instructions;
6993 }
6994
6995 static uint32_t
6996 instr_optimize(struct rte_swx_pipeline *p,
6997                struct action *a,
6998                struct instruction *instructions,
6999                struct instruction_data *instruction_data,
7000                uint32_t n_instructions)
7001 {
7002         /* Extract many. */
7003         n_instructions = instr_pattern_extract_many_optimize(instructions,
7004                                                              instruction_data,
7005                                                              n_instructions);
7006
7007         /* Emit many + TX. */
7008         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7009                                                              instruction_data,
7010                                                              n_instructions);
7011
7012         /* Validate + mov all. */
7013         n_instructions = instr_pattern_validate_mov_all_optimize(p,
7014                                                                  a,
7015                                                                  instructions,
7016                                                                  instruction_data,
7017                                                                  n_instructions);
7018
7019         /* DMA many. */
7020         n_instructions = instr_pattern_dma_many_optimize(instructions,
7021                                                          instruction_data,
7022                                                          n_instructions);
7023
7024         return n_instructions;
7025 }
7026
7027 static int
7028 instruction_config(struct rte_swx_pipeline *p,
7029                    struct action *a,
7030                    const char **instructions,
7031                    uint32_t n_instructions)
7032 {
7033         struct instruction *instr = NULL;
7034         struct instruction_data *data = NULL;
7035         int err = 0;
7036         uint32_t i;
7037
7038         CHECK(n_instructions, EINVAL);
7039         CHECK(instructions, EINVAL);
7040         for (i = 0; i < n_instructions; i++)
7041                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7042
7043         /* Memory allocation. */
7044         instr = calloc(n_instructions, sizeof(struct instruction));
7045         if (!instr) {
7046                 err = -ENOMEM;
7047                 goto error;
7048         }
7049
7050         data = calloc(n_instructions, sizeof(struct instruction_data));
7051         if (!data) {
7052                 err = -ENOMEM;
7053                 goto error;
7054         }
7055
7056         for (i = 0; i < n_instructions; i++) {
7057                 char *string = strdup(instructions[i]);
7058                 if (!string) {
7059                         err = -ENOMEM;
7060                         goto error;
7061                 }
7062
7063                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7064                 if (err) {
7065                         free(string);
7066                         goto error;
7067                 }
7068
7069                 free(string);
7070         }
7071
7072         err = instr_label_check(data, n_instructions);
7073         if (err)
7074                 goto error;
7075
7076         err = instr_verify(p, a, instr, data, n_instructions);
7077         if (err)
7078                 goto error;
7079
7080         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7081
7082         err = instr_jmp_resolve(instr, data, n_instructions);
7083         if (err)
7084                 goto error;
7085
7086         if (a) {
7087                 a->instructions = instr;
7088                 a->instruction_data = data;
7089                 a->n_instructions = n_instructions;
7090         } else {
7091                 p->instructions = instr;
7092                 p->instruction_data = data;
7093                 p->n_instructions = n_instructions;
7094         }
7095
7096         return 0;
7097
7098 error:
7099         free(data);
7100         free(instr);
7101         return err;
7102 }
7103
7104 static instr_exec_t instruction_table[] = {
7105         [INSTR_RX] = instr_rx_exec,
7106         [INSTR_TX] = instr_tx_exec,
7107         [INSTR_TX_I] = instr_tx_i_exec,
7108         [INSTR_DROP] = instr_drop_exec,
7109         [INSTR_MIRROR] = instr_mirror_exec,
7110         [INSTR_RECIRCULATE] = instr_recirculate_exec,
7111         [INSTR_RECIRCID] = instr_recircid_exec,
7112
7113         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7114         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7115         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7116         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7117         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7118         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7119         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7120         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7121         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7122         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7123
7124         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7125         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7126         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7127         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7128         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7129         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7130         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7131         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7132         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7133
7134         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7135         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7136
7137         [INSTR_MOV] = instr_mov_exec,
7138         [INSTR_MOV_MH] = instr_mov_mh_exec,
7139         [INSTR_MOV_HM] = instr_mov_hm_exec,
7140         [INSTR_MOV_HH] = instr_mov_hh_exec,
7141         [INSTR_MOV_I] = instr_mov_i_exec,
7142
7143         [INSTR_DMA_HT] = instr_dma_ht_exec,
7144         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7145         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7146         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7147         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7148         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7149         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7150         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7151
7152         [INSTR_ALU_ADD] = instr_alu_add_exec,
7153         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7154         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7155         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7156         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7157         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7158
7159         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7160         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7161         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7162         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7163         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7164         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7165
7166         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7167         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7168         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7169         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7170
7171         [INSTR_ALU_AND] = instr_alu_and_exec,
7172         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7173         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7174         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7175         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7176
7177         [INSTR_ALU_OR] = instr_alu_or_exec,
7178         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7179         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7180         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7181         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7182
7183         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7184         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7185         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7186         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7187         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7188
7189         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7190         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7191         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7192         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7193         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7194         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7195
7196         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7197         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7198         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7199         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7200         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7201         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7202
7203         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7204         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7205         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7206
7207         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7208         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7209         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7210         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7211         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7212         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7213
7214         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7215         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7216         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7217         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7218         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7219         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7220         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7221         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7222         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7223
7224         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7225         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7226         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7227         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7228         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7229         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7230         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7231         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7232         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7233
7234         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7235         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7236         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7237
7238         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7239         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7240         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7241         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7242         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7243         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7244         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7245         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7246         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7247         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7248         [INSTR_METER_IMM] = instr_meter_imm_exec,
7249         [INSTR_METER_IMI] = instr_meter_imi_exec,
7250
7251         [INSTR_TABLE] = instr_table_exec,
7252         [INSTR_TABLE_AF] = instr_table_af_exec,
7253         [INSTR_SELECTOR] = instr_selector_exec,
7254         [INSTR_LEARNER] = instr_learner_exec,
7255         [INSTR_LEARNER_AF] = instr_learner_af_exec,
7256         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7257         [INSTR_LEARNER_REARM] = instr_rearm_exec,
7258         [INSTR_LEARNER_REARM_NEW] = instr_rearm_new_exec,
7259         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7260         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7261         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7262         [INSTR_HASH_FUNC] = instr_hash_func_exec,
7263
7264         [INSTR_JMP] = instr_jmp_exec,
7265         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7266         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7267         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7268         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7269         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7270         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7271
7272         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7273         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7274         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7275         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7276         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7277
7278         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7279         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7280         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7281         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7282         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7283
7284         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7285         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7286         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7287         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7288         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7289         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7290
7291         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7292         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7293         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7294         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7295         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7296         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7297
7298         [INSTR_RETURN] = instr_return_exec,
7299 };
7300
7301 static int
7302 instruction_table_build(struct rte_swx_pipeline *p)
7303 {
7304         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
7305                                       sizeof(struct instr_exec_t *));
7306         if (!p->instruction_table)
7307                 return -EINVAL;
7308
7309         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
7310
7311         return 0;
7312 }
7313
7314 static void
7315 instruction_table_build_free(struct rte_swx_pipeline *p)
7316 {
7317         if (!p->instruction_table)
7318                 return;
7319
7320         free(p->instruction_table);
7321         p->instruction_table = NULL;
7322 }
7323
7324 static void
7325 instruction_table_free(struct rte_swx_pipeline *p)
7326 {
7327         instruction_table_build_free(p);
7328 }
7329
7330 static inline void
7331 instr_exec(struct rte_swx_pipeline *p)
7332 {
7333         struct thread *t = &p->threads[p->thread_id];
7334         struct instruction *ip = t->ip;
7335         instr_exec_t instr = p->instruction_table[ip->type];
7336
7337         instr(p);
7338 }
7339
7340 /*
7341  * Action.
7342  */
7343 static struct action *
7344 action_find(struct rte_swx_pipeline *p, const char *name)
7345 {
7346         struct action *elem;
7347
7348         if (!name)
7349                 return NULL;
7350
7351         TAILQ_FOREACH(elem, &p->actions, node)
7352                 if (strcmp(elem->name, name) == 0)
7353                         return elem;
7354
7355         return NULL;
7356 }
7357
7358 static struct action *
7359 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7360 {
7361         struct action *action = NULL;
7362
7363         TAILQ_FOREACH(action, &p->actions, node)
7364                 if (action->id == id)
7365                         return action;
7366
7367         return NULL;
7368 }
7369
7370 static struct field *
7371 action_field_find(struct action *a, const char *name)
7372 {
7373         return a->st ? struct_type_field_find(a->st, name) : NULL;
7374 }
7375
7376 static struct field *
7377 action_field_parse(struct action *action, const char *name)
7378 {
7379         if (name[0] != 't' || name[1] != '.')
7380                 return NULL;
7381
7382         return action_field_find(action, &name[2]);
7383 }
7384
7385 static int
7386 action_has_nbo_args(struct action *a)
7387 {
7388         uint32_t i;
7389
7390         /* Return if the action does not have any args. */
7391         if (!a->st)
7392                 return 0; /* FALSE */
7393
7394         for (i = 0; i < a->st->n_fields; i++)
7395                 if (a->args_endianness[i])
7396                         return 1; /* TRUE */
7397
7398         return 0; /* FALSE */
7399 }
7400
7401 static int
7402 action_does_learning(struct action *a)
7403 {
7404         uint32_t i;
7405
7406         for (i = 0; i < a->n_instructions; i++)
7407                 switch (a->instructions[i].type) {
7408                 case INSTR_LEARNER_LEARN:
7409                         return 1; /* TRUE */
7410
7411                 case INSTR_LEARNER_FORGET:
7412                         return 1; /* TRUE */
7413
7414                 default:
7415                         continue;
7416                 }
7417
7418         return 0; /* FALSE */
7419 }
7420
7421 int
7422 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7423                                const char *name,
7424                                const char *args_struct_type_name,
7425                                const char **instructions,
7426                                uint32_t n_instructions)
7427 {
7428         struct struct_type *args_struct_type = NULL;
7429         struct action *a = NULL;
7430         int status = 0;
7431
7432         CHECK(p, EINVAL);
7433
7434         CHECK_NAME(name, EINVAL);
7435         CHECK(!action_find(p, name), EEXIST);
7436
7437         if (args_struct_type_name) {
7438                 CHECK_NAME(args_struct_type_name, EINVAL);
7439                 args_struct_type = struct_type_find(p, args_struct_type_name);
7440                 CHECK(args_struct_type, EINVAL);
7441                 CHECK(!args_struct_type->var_size, EINVAL);
7442         }
7443
7444         /* Node allocation. */
7445         a = calloc(1, sizeof(struct action));
7446         if (!a) {
7447                 status = -ENOMEM;
7448                 goto error;
7449         }
7450
7451         if (args_struct_type) {
7452                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7453                 if (!a->args_endianness) {
7454                         status = -ENOMEM;
7455                         goto error;
7456                 }
7457         }
7458
7459         /* Node initialization. */
7460         strcpy(a->name, name);
7461         a->st = args_struct_type;
7462         a->id = p->n_actions;
7463
7464         /* Instruction translation. */
7465         status = instruction_config(p, a, instructions, n_instructions);
7466         if (status)
7467                 goto error;
7468
7469         /* Node add to tailq. */
7470         TAILQ_INSERT_TAIL(&p->actions, a, node);
7471         p->n_actions++;
7472
7473         return 0;
7474
7475 error:
7476         if (!a)
7477                 return status;
7478
7479         free(a->args_endianness);
7480         free(a->instructions);
7481         free(a->instruction_data);
7482         free(a);
7483
7484         return status;
7485 }
7486
7487 static int
7488 action_build(struct rte_swx_pipeline *p)
7489 {
7490         struct action *action;
7491
7492         /* p->action_instructions. */
7493         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7494         CHECK(p->action_instructions, ENOMEM);
7495
7496         TAILQ_FOREACH(action, &p->actions, node)
7497                 p->action_instructions[action->id] = action->instructions;
7498
7499         /* p->action_funcs. */
7500         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7501         CHECK(p->action_funcs, ENOMEM);
7502
7503         return 0;
7504 }
7505
7506 static void
7507 action_build_free(struct rte_swx_pipeline *p)
7508 {
7509         free(p->action_funcs);
7510         p->action_funcs = NULL;
7511
7512         free(p->action_instructions);
7513         p->action_instructions = NULL;
7514 }
7515
7516 static void
7517 action_free(struct rte_swx_pipeline *p)
7518 {
7519         action_build_free(p);
7520
7521         for ( ; ; ) {
7522                 struct action *action;
7523
7524                 action = TAILQ_FIRST(&p->actions);
7525                 if (!action)
7526                         break;
7527
7528                 TAILQ_REMOVE(&p->actions, action, node);
7529                 free(action->args_endianness);
7530                 free(action->instructions);
7531                 free(action->instruction_data);
7532                 free(action);
7533         }
7534 }
7535
7536 static uint32_t
7537 action_arg_src_mov_count(struct action *a,
7538                          uint32_t arg_id,
7539                          struct instruction *instructions,
7540                          struct instruction_data *instruction_data,
7541                          uint32_t n_instructions)
7542 {
7543         uint32_t offset, n_users = 0, i;
7544
7545         if (!a->st ||
7546             (arg_id >= a->st->n_fields) ||
7547             !instructions ||
7548             !instruction_data ||
7549             !n_instructions)
7550                 return 0;
7551
7552         offset = a->st->fields[arg_id].offset / 8;
7553
7554         for (i = 0; i < n_instructions; i++) {
7555                 struct instruction *instr = &instructions[i];
7556                 struct instruction_data *data = &instruction_data[i];
7557
7558                 if (data->invalid ||
7559                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7560                     instr->mov.src.struct_id ||
7561                     (instr->mov.src.offset != offset))
7562                         continue;
7563
7564                 n_users++;
7565         }
7566
7567         return n_users;
7568 }
7569
7570 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7571 #define field_ntoh(val, n_bits) (ntoh64((val) << (64 - n_bits)))
7572 #define field_hton(val, n_bits) (hton64((val) << (64 - n_bits)))
7573 #else
7574 #define field_ntoh(val, n_bits) (val)
7575 #define field_hton(val, n_bits) (val)
7576 #endif
7577
7578 #define ACTION_ARGS_TOKENS_MAX 256
7579
7580 static int
7581 action_args_parse(struct action *a, const char *args, uint8_t *data)
7582 {
7583         char *tokens[ACTION_ARGS_TOKENS_MAX], *s0 = NULL, *s;
7584         uint32_t n_tokens = 0, offset = 0, i;
7585         int status = 0;
7586
7587         /* Checks. */
7588         if (!a->st || !args || !args[0]) {
7589                 status = -EINVAL;
7590                 goto error;
7591         }
7592
7593         /* Memory allocation. */
7594         s0 = strdup(args);
7595         if (!s0) {
7596                 status = -ENOMEM;
7597                 goto error;
7598         }
7599
7600         /* Parse the string into tokens. */
7601         for (s = s0; ; ) {
7602                 char *token;
7603
7604                 token = strtok_r(s, " \f\n\r\t\v", &s);
7605                 if (!token)
7606                         break;
7607
7608                 if (n_tokens >= RTE_DIM(tokens)) {
7609                         status = -EINVAL;
7610                         goto error;
7611                 }
7612
7613                 tokens[n_tokens] = token;
7614                 n_tokens++;
7615         }
7616
7617         /* More checks. */
7618         if (n_tokens != a->st->n_fields * 2) {
7619                 status = -EINVAL;
7620                 goto error;
7621         }
7622
7623         /* Process the action arguments. */
7624         for (i = 0; i < a->st->n_fields; i++) {
7625                 struct field *f = &a->st->fields[i];
7626                 char *arg_name = tokens[i * 2];
7627                 char *arg_val = tokens[i * 2 + 1];
7628                 uint64_t val;
7629
7630                 if (strcmp(arg_name, f->name)) {
7631                         status = -EINVAL;
7632                         goto error;
7633                 }
7634
7635                 val = strtoull(arg_val, &arg_val, 0);
7636                 if (arg_val[0]) {
7637                         status = -EINVAL;
7638                         goto error;
7639                 }
7640
7641                 /* Endianness conversion. */
7642                 if (a->args_endianness[i])
7643                         val = field_hton(val, f->n_bits);
7644
7645                 /* Copy to entry. */
7646                 memcpy(&data[offset], (uint8_t *)&val, f->n_bits / 8);
7647                 offset += f->n_bits / 8;
7648         }
7649
7650 error:
7651         free(s0);
7652         return status;
7653 }
7654
7655 /*
7656  * Table.
7657  */
7658 static struct table_type *
7659 table_type_find(struct rte_swx_pipeline *p, const char *name)
7660 {
7661         struct table_type *elem;
7662
7663         TAILQ_FOREACH(elem, &p->table_types, node)
7664                 if (strcmp(elem->name, name) == 0)
7665                         return elem;
7666
7667         return NULL;
7668 }
7669
7670 static struct table_type *
7671 table_type_resolve(struct rte_swx_pipeline *p,
7672                    const char *recommended_type_name,
7673                    enum rte_swx_table_match_type match_type)
7674 {
7675         struct table_type *elem;
7676
7677         /* Only consider the recommended type if the match type is correct. */
7678         if (recommended_type_name)
7679                 TAILQ_FOREACH(elem, &p->table_types, node)
7680                         if (!strcmp(elem->name, recommended_type_name) &&
7681                             (elem->match_type == match_type))
7682                                 return elem;
7683
7684         /* Ignore the recommended type and get the first element with this match
7685          * type.
7686          */
7687         TAILQ_FOREACH(elem, &p->table_types, node)
7688                 if (elem->match_type == match_type)
7689                         return elem;
7690
7691         return NULL;
7692 }
7693
7694 static struct table *
7695 table_find(struct rte_swx_pipeline *p, const char *name)
7696 {
7697         struct table *elem;
7698
7699         TAILQ_FOREACH(elem, &p->tables, node)
7700                 if (strcmp(elem->name, name) == 0)
7701                         return elem;
7702
7703         return NULL;
7704 }
7705
7706 static struct table *
7707 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7708 {
7709         struct table *table = NULL;
7710
7711         TAILQ_FOREACH(table, &p->tables, node)
7712                 if (table->id == id)
7713                         return table;
7714
7715         return NULL;
7716 }
7717
7718 int
7719 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7720                                      const char *name,
7721                                      enum rte_swx_table_match_type match_type,
7722                                      struct rte_swx_table_ops *ops)
7723 {
7724         struct table_type *elem;
7725
7726         CHECK(p, EINVAL);
7727
7728         CHECK_NAME(name, EINVAL);
7729         CHECK(!table_type_find(p, name), EEXIST);
7730
7731         CHECK(ops, EINVAL);
7732         CHECK(ops->create, EINVAL);
7733         CHECK(ops->lkp, EINVAL);
7734         CHECK(ops->free, EINVAL);
7735
7736         /* Node allocation. */
7737         elem = calloc(1, sizeof(struct table_type));
7738         CHECK(elem, ENOMEM);
7739
7740         /* Node initialization. */
7741         strcpy(elem->name, name);
7742         elem->match_type = match_type;
7743         memcpy(&elem->ops, ops, sizeof(*ops));
7744
7745         /* Node add to tailq. */
7746         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7747
7748         return 0;
7749 }
7750
7751 static int
7752 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7753                          uint32_t n_fields,
7754                          enum rte_swx_table_match_type *match_type)
7755 {
7756         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7757
7758         for (i = 0; i < n_fields; i++) {
7759                 struct rte_swx_match_field_params  *f = &fields[i];
7760
7761                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7762                         n_fields_em++;
7763
7764                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7765                         n_fields_lpm++;
7766         }
7767
7768         if ((n_fields_lpm > 1) ||
7769             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7770                 return -EINVAL;
7771
7772         *match_type = (n_fields_em == n_fields) ?
7773                        RTE_SWX_TABLE_MATCH_EXACT :
7774                        RTE_SWX_TABLE_MATCH_WILDCARD;
7775
7776         return 0;
7777 }
7778
7779 static int
7780 table_match_fields_check(struct rte_swx_pipeline *p,
7781                          struct rte_swx_pipeline_table_params *params,
7782                          struct header **header)
7783 {
7784         struct header *h0 = NULL;
7785         struct field *hf, *mf;
7786         uint32_t *offset = NULL, i;
7787         int status = 0;
7788
7789         /* Return if no match fields. */
7790         if (!params->n_fields) {
7791                 if (params->fields) {
7792                         status = -EINVAL;
7793                         goto end;
7794                 }
7795
7796                 if (header)
7797                         *header = NULL;
7798
7799                 return 0;
7800         }
7801
7802         /* Memory allocation. */
7803         offset = calloc(params->n_fields, sizeof(uint32_t));
7804         if (!offset) {
7805                 status = -ENOMEM;
7806                 goto end;
7807         }
7808
7809         /* Check that all the match fields belong to either the same header or
7810          * to the meta-data.
7811          */
7812         hf = header_field_parse(p, params->fields[0].name, &h0);
7813         mf = metadata_field_parse(p, params->fields[0].name);
7814         if ((!hf && !mf) || (hf && hf->var_size)) {
7815                 status = -EINVAL;
7816                 goto end;
7817         }
7818
7819         offset[0] = h0 ? hf->offset : mf->offset;
7820
7821         for (i = 1; i < params->n_fields; i++)
7822                 if (h0) {
7823                         struct header *h;
7824
7825                         hf = header_field_parse(p, params->fields[i].name, &h);
7826                         if (!hf || (h->id != h0->id) || hf->var_size) {
7827                                 status = -EINVAL;
7828                                 goto end;
7829                         }
7830
7831                         offset[i] = hf->offset;
7832                 } else {
7833                         mf = metadata_field_parse(p, params->fields[i].name);
7834                         if (!mf) {
7835                                 status = -EINVAL;
7836                                 goto end;
7837                         }
7838
7839                         offset[i] = mf->offset;
7840                 }
7841
7842         /* Check that there are no duplicated match fields. */
7843         for (i = 0; i < params->n_fields; i++) {
7844                 uint32_t j;
7845
7846                 for (j = 0; j < i; j++)
7847                         if (offset[j] == offset[i]) {
7848                                 status = -EINVAL;
7849                                 goto end;
7850                         }
7851         }
7852
7853         /* Return. */
7854         if (header)
7855                 *header = h0;
7856
7857 end:
7858         free(offset);
7859         return status;
7860 }
7861
7862 int
7863 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7864                               const char *name,
7865                               struct rte_swx_pipeline_table_params *params,
7866                               const char *recommended_table_type_name,
7867                               const char *args,
7868                               uint32_t size)
7869 {
7870         struct table_type *type;
7871         struct table *t = NULL;
7872         struct action *default_action;
7873         struct header *header = NULL;
7874         uint32_t action_data_size_max = 0, i;
7875         int status = 0;
7876
7877         CHECK(p, EINVAL);
7878
7879         CHECK_NAME(name, EINVAL);
7880         CHECK(!table_find(p, name), EEXIST);
7881         CHECK(!selector_find(p, name), EEXIST);
7882         CHECK(!learner_find(p, name), EEXIST);
7883
7884         CHECK(params, EINVAL);
7885
7886         /* Match checks. */
7887         status = table_match_fields_check(p, params, &header);
7888         if (status)
7889                 return status;
7890
7891         /* Action checks. */
7892         CHECK(params->n_actions, EINVAL);
7893         CHECK(params->action_names, EINVAL);
7894         for (i = 0; i < params->n_actions; i++) {
7895                 const char *action_name = params->action_names[i];
7896                 struct action *a;
7897                 uint32_t action_data_size;
7898                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7899
7900                 CHECK_NAME(action_name, EINVAL);
7901
7902                 a = action_find(p, action_name);
7903                 CHECK(a, EINVAL);
7904                 CHECK(!action_does_learning(a), EINVAL);
7905
7906                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7907                 if (action_data_size > action_data_size_max)
7908                         action_data_size_max = action_data_size;
7909
7910                 if (params->action_is_for_table_entries)
7911                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7912                 if (params->action_is_for_default_entry)
7913                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7914                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7915         }
7916
7917         CHECK_NAME(params->default_action_name, EINVAL);
7918         for (i = 0; i < p->n_actions; i++)
7919                 if (!strcmp(params->action_names[i],
7920                             params->default_action_name))
7921                         break;
7922         CHECK(i < params->n_actions, EINVAL);
7923         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7924               EINVAL);
7925
7926         default_action = action_find(p, params->default_action_name);
7927         CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
7928               EINVAL);
7929
7930         /* Table type checks. */
7931         if (recommended_table_type_name)
7932                 CHECK_NAME(recommended_table_type_name, EINVAL);
7933
7934         if (params->n_fields) {
7935                 enum rte_swx_table_match_type match_type;
7936
7937                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7938                 if (status)
7939                         return status;
7940
7941                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7942                 CHECK(type, EINVAL);
7943         } else {
7944                 type = NULL;
7945         }
7946
7947         /* Memory allocation. */
7948         t = calloc(1, sizeof(struct table));
7949         if (!t) {
7950                 status = -ENOMEM;
7951                 goto error;
7952         }
7953
7954         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7955         if (!t->fields) {
7956                 status = -ENOMEM;
7957                 goto error;
7958         }
7959
7960         t->actions = calloc(params->n_actions, sizeof(struct action *));
7961         if (!t->actions) {
7962                 status = -ENOMEM;
7963                 goto error;
7964         }
7965
7966         if (action_data_size_max) {
7967                 t->default_action_data = calloc(1, action_data_size_max);
7968                 if (!t->default_action_data) {
7969                         status = -ENOMEM;
7970                         goto error;
7971                 }
7972         }
7973
7974         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7975         if (!t->action_is_for_table_entries) {
7976                 status = -ENOMEM;
7977                 goto error;
7978         }
7979
7980         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7981         if (!t->action_is_for_default_entry) {
7982                 status = -ENOMEM;
7983                 goto error;
7984         }
7985
7986         /* Node initialization. */
7987         strcpy(t->name, name);
7988         if (args && args[0])
7989                 strcpy(t->args, args);
7990         t->type = type;
7991
7992         for (i = 0; i < params->n_fields; i++) {
7993                 struct rte_swx_match_field_params *field = &params->fields[i];
7994                 struct match_field *f = &t->fields[i];
7995
7996                 f->match_type = field->match_type;
7997                 f->field = header ?
7998                         header_field_parse(p, field->name, NULL) :
7999                         metadata_field_parse(p, field->name);
8000         }
8001         t->n_fields = params->n_fields;
8002         t->header = header;
8003
8004         for (i = 0; i < params->n_actions; i++) {
8005                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8006
8007                 if (params->action_is_for_table_entries)
8008                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8009                 if (params->action_is_for_default_entry)
8010                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8011
8012                 t->actions[i] = action_find(p, params->action_names[i]);
8013                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
8014                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
8015         }
8016         t->default_action = default_action;
8017         if (default_action->st) {
8018                 status = action_args_parse(default_action,
8019                                            params->default_action_args,
8020                                            t->default_action_data);
8021                 if (status)
8022                         goto error;
8023         }
8024
8025         t->n_actions = params->n_actions;
8026         t->default_action_is_const = params->default_action_is_const;
8027         t->action_data_size_max = action_data_size_max;
8028
8029         t->size = size;
8030         t->id = p->n_tables;
8031
8032         /* Node add to tailq. */
8033         TAILQ_INSERT_TAIL(&p->tables, t, node);
8034         p->n_tables++;
8035
8036         return 0;
8037
8038 error:
8039         if (!t)
8040                 return status;
8041
8042         free(t->action_is_for_default_entry);
8043         free(t->action_is_for_table_entries);
8044         free(t->default_action_data);
8045         free(t->actions);
8046         free(t->fields);
8047         free(t);
8048
8049         return status;
8050 }
8051
8052 static struct rte_swx_table_params *
8053 table_params_get(struct table *table)
8054 {
8055         struct rte_swx_table_params *params;
8056         struct field *first, *last;
8057         uint8_t *key_mask;
8058         uint32_t key_size, key_offset, action_data_size, i;
8059
8060         /* Memory allocation. */
8061         params = calloc(1, sizeof(struct rte_swx_table_params));
8062         if (!params)
8063                 return NULL;
8064
8065         /* Find first (smallest offset) and last (biggest offset) match fields. */
8066         first = table->fields[0].field;
8067         last = table->fields[0].field;
8068
8069         for (i = 0; i < table->n_fields; i++) {
8070                 struct field *f = table->fields[i].field;
8071
8072                 if (f->offset < first->offset)
8073                         first = f;
8074
8075                 if (f->offset > last->offset)
8076                         last = f;
8077         }
8078
8079         /* Key offset and size. */
8080         key_offset = first->offset / 8;
8081         key_size = (last->offset + last->n_bits - first->offset) / 8;
8082
8083         /* Memory allocation. */
8084         key_mask = calloc(1, key_size);
8085         if (!key_mask) {
8086                 free(params);
8087                 return NULL;
8088         }
8089
8090         /* Key mask. */
8091         for (i = 0; i < table->n_fields; i++) {
8092                 struct field *f = table->fields[i].field;
8093                 uint32_t start = (f->offset - first->offset) / 8;
8094                 size_t size = f->n_bits / 8;
8095
8096                 memset(&key_mask[start], 0xFF, size);
8097         }
8098
8099         /* Action data size. */
8100         action_data_size = 0;
8101         for (i = 0; i < table->n_actions; i++) {
8102                 struct action *action = table->actions[i];
8103                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8104
8105                 if (ads > action_data_size)
8106                         action_data_size = ads;
8107         }
8108
8109         /* Fill in. */
8110         params->match_type = table->type->match_type;
8111         params->key_size = key_size;
8112         params->key_offset = key_offset;
8113         params->key_mask0 = key_mask;
8114         params->action_data_size = action_data_size;
8115         params->n_keys_max = table->size;
8116
8117         return params;
8118 }
8119
8120 static void
8121 table_params_free(struct rte_swx_table_params *params)
8122 {
8123         if (!params)
8124                 return;
8125
8126         free(params->key_mask0);
8127         free(params);
8128 }
8129
8130 static int
8131 table_stub_lkp(void *table __rte_unused,
8132                void *mailbox __rte_unused,
8133                uint8_t **key __rte_unused,
8134                uint64_t *action_id __rte_unused,
8135                uint8_t **action_data __rte_unused,
8136                int *hit)
8137 {
8138         *hit = 0;
8139         return 1; /* DONE. */
8140 }
8141
8142 static int
8143 table_build(struct rte_swx_pipeline *p)
8144 {
8145         uint32_t i;
8146
8147         /* Per pipeline: table statistics. */
8148         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8149         CHECK(p->table_stats, ENOMEM);
8150
8151         for (i = 0; i < p->n_tables; i++) {
8152                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8153                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8154         }
8155
8156         /* Per thread: table runt-time. */
8157         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8158                 struct thread *t = &p->threads[i];
8159                 struct table *table;
8160
8161                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8162                 CHECK(t->tables, ENOMEM);
8163
8164                 TAILQ_FOREACH(table, &p->tables, node) {
8165                         struct table_runtime *r = &t->tables[table->id];
8166
8167                         if (table->type) {
8168                                 uint64_t size;
8169
8170                                 size = table->type->ops.mailbox_size_get();
8171
8172                                 /* r->func. */
8173                                 r->func = table->type->ops.lkp;
8174
8175                                 /* r->mailbox. */
8176                                 if (size) {
8177                                         r->mailbox = calloc(1, size);
8178                                         CHECK(r->mailbox, ENOMEM);
8179                                 }
8180
8181                                 /* r->key. */
8182                                 r->key = table->header ?
8183                                         &t->structs[table->header->struct_id] :
8184                                         &t->structs[p->metadata_struct_id];
8185                         } else {
8186                                 r->func = table_stub_lkp;
8187                         }
8188                 }
8189         }
8190
8191         return 0;
8192 }
8193
8194 static void
8195 table_build_free(struct rte_swx_pipeline *p)
8196 {
8197         uint32_t i;
8198
8199         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8200                 struct thread *t = &p->threads[i];
8201                 uint32_t j;
8202
8203                 if (!t->tables)
8204                         continue;
8205
8206                 for (j = 0; j < p->n_tables; j++) {
8207                         struct table_runtime *r = &t->tables[j];
8208
8209                         free(r->mailbox);
8210                 }
8211
8212                 free(t->tables);
8213                 t->tables = NULL;
8214         }
8215
8216         if (p->table_stats) {
8217                 for (i = 0; i < p->n_tables; i++)
8218                         free(p->table_stats[i].n_pkts_action);
8219
8220                 free(p->table_stats);
8221         }
8222 }
8223
8224 static void
8225 table_free(struct rte_swx_pipeline *p)
8226 {
8227         table_build_free(p);
8228
8229         /* Tables. */
8230         for ( ; ; ) {
8231                 struct table *elem;
8232
8233                 elem = TAILQ_FIRST(&p->tables);
8234                 if (!elem)
8235                         break;
8236
8237                 TAILQ_REMOVE(&p->tables, elem, node);
8238                 free(elem->fields);
8239                 free(elem->actions);
8240                 free(elem->default_action_data);
8241                 free(elem);
8242         }
8243
8244         /* Table types. */
8245         for ( ; ; ) {
8246                 struct table_type *elem;
8247
8248                 elem = TAILQ_FIRST(&p->table_types);
8249                 if (!elem)
8250                         break;
8251
8252                 TAILQ_REMOVE(&p->table_types, elem, node);
8253                 free(elem);
8254         }
8255 }
8256
8257 /*
8258  * Selector.
8259  */
8260 static struct selector *
8261 selector_find(struct rte_swx_pipeline *p, const char *name)
8262 {
8263         struct selector *s;
8264
8265         TAILQ_FOREACH(s, &p->selectors, node)
8266                 if (strcmp(s->name, name) == 0)
8267                         return s;
8268
8269         return NULL;
8270 }
8271
8272 static struct selector *
8273 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8274 {
8275         struct selector *s = NULL;
8276
8277         TAILQ_FOREACH(s, &p->selectors, node)
8278                 if (s->id == id)
8279                         return s;
8280
8281         return NULL;
8282 }
8283
8284 static int
8285 selector_fields_check(struct rte_swx_pipeline *p,
8286                       struct rte_swx_pipeline_selector_params *params,
8287                       struct header **header)
8288 {
8289         struct header *h0 = NULL;
8290         struct field *hf, *mf;
8291         uint32_t i;
8292
8293         /* Return if no selector fields. */
8294         if (!params->n_selector_fields || !params->selector_field_names)
8295                 return -EINVAL;
8296
8297         /* Check that all the selector fields either belong to the same header
8298          * or are all meta-data fields.
8299          */
8300         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8301         mf = metadata_field_parse(p, params->selector_field_names[0]);
8302         if (!hf && !mf)
8303                 return -EINVAL;
8304
8305         for (i = 1; i < params->n_selector_fields; i++)
8306                 if (h0) {
8307                         struct header *h;
8308
8309                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8310                         if (!hf || (h->id != h0->id))
8311                                 return -EINVAL;
8312                 } else {
8313                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8314                         if (!mf)
8315                                 return -EINVAL;
8316                 }
8317
8318         /* Check that there are no duplicated match fields. */
8319         for (i = 0; i < params->n_selector_fields; i++) {
8320                 const char *field_name = params->selector_field_names[i];
8321                 uint32_t j;
8322
8323                 for (j = i + 1; j < params->n_selector_fields; j++)
8324                         if (!strcmp(params->selector_field_names[j], field_name))
8325                                 return -EINVAL;
8326         }
8327
8328         /* Return. */
8329         if (header)
8330                 *header = h0;
8331
8332         return 0;
8333 }
8334
8335 int
8336 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8337                                  const char *name,
8338                                  struct rte_swx_pipeline_selector_params *params)
8339 {
8340         struct selector *s;
8341         struct header *selector_header = NULL;
8342         struct field *group_id_field, *member_id_field;
8343         uint32_t i;
8344         int status = 0;
8345
8346         CHECK(p, EINVAL);
8347
8348         CHECK_NAME(name, EINVAL);
8349         CHECK(!table_find(p, name), EEXIST);
8350         CHECK(!selector_find(p, name), EEXIST);
8351         CHECK(!learner_find(p, name), EEXIST);
8352
8353         CHECK(params, EINVAL);
8354
8355         CHECK_NAME(params->group_id_field_name, EINVAL);
8356         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8357         CHECK(group_id_field, EINVAL);
8358
8359         for (i = 0; i < params->n_selector_fields; i++) {
8360                 const char *field_name = params->selector_field_names[i];
8361
8362                 CHECK_NAME(field_name, EINVAL);
8363         }
8364         status = selector_fields_check(p, params, &selector_header);
8365         if (status)
8366                 return status;
8367
8368         CHECK_NAME(params->member_id_field_name, EINVAL);
8369         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8370         CHECK(member_id_field, EINVAL);
8371
8372         CHECK(params->n_groups_max, EINVAL);
8373
8374         CHECK(params->n_members_per_group_max, EINVAL);
8375
8376         /* Memory allocation. */
8377         s = calloc(1, sizeof(struct selector));
8378         if (!s) {
8379                 status = -ENOMEM;
8380                 goto error;
8381         }
8382
8383         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8384         if (!s->selector_fields) {
8385                 status = -ENOMEM;
8386                 goto error;
8387         }
8388
8389         /* Node initialization. */
8390         strcpy(s->name, name);
8391
8392         s->group_id_field = group_id_field;
8393
8394         for (i = 0; i < params->n_selector_fields; i++) {
8395                 const char *field_name = params->selector_field_names[i];
8396
8397                 s->selector_fields[i] = selector_header ?
8398                         header_field_parse(p, field_name, NULL) :
8399                         metadata_field_parse(p, field_name);
8400         }
8401
8402         s->n_selector_fields = params->n_selector_fields;
8403
8404         s->selector_header = selector_header;
8405
8406         s->member_id_field = member_id_field;
8407
8408         s->n_groups_max = params->n_groups_max;
8409
8410         s->n_members_per_group_max = params->n_members_per_group_max;
8411
8412         s->id = p->n_selectors;
8413
8414         /* Node add to tailq. */
8415         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8416         p->n_selectors++;
8417
8418         return 0;
8419
8420 error:
8421         if (!s)
8422                 return status;
8423
8424         free(s->selector_fields);
8425
8426         free(s);
8427
8428         return status;
8429 }
8430
8431 static void
8432 selector_params_free(struct rte_swx_table_selector_params *params)
8433 {
8434         if (!params)
8435                 return;
8436
8437         free(params->selector_mask);
8438
8439         free(params);
8440 }
8441
8442 static struct rte_swx_table_selector_params *
8443 selector_table_params_get(struct selector *s)
8444 {
8445         struct rte_swx_table_selector_params *params = NULL;
8446         struct field *first, *last;
8447         uint32_t i;
8448
8449         /* Memory allocation. */
8450         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8451         if (!params)
8452                 goto error;
8453
8454         /* Group ID. */
8455         params->group_id_offset = s->group_id_field->offset / 8;
8456
8457         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8458         first = s->selector_fields[0];
8459         last = s->selector_fields[0];
8460
8461         for (i = 0; i < s->n_selector_fields; i++) {
8462                 struct field *f = s->selector_fields[i];
8463
8464                 if (f->offset < first->offset)
8465                         first = f;
8466
8467                 if (f->offset > last->offset)
8468                         last = f;
8469         }
8470
8471         /* Selector offset and size. */
8472         params->selector_offset = first->offset / 8;
8473         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8474
8475         /* Memory allocation. */
8476         params->selector_mask = calloc(1, params->selector_size);
8477         if (!params->selector_mask)
8478                 goto error;
8479
8480         /* Selector mask. */
8481         for (i = 0; i < s->n_selector_fields; i++) {
8482                 struct field *f = s->selector_fields[i];
8483                 uint32_t start = (f->offset - first->offset) / 8;
8484                 size_t size = f->n_bits / 8;
8485
8486                 memset(&params->selector_mask[start], 0xFF, size);
8487         }
8488
8489         /* Member ID. */
8490         params->member_id_offset = s->member_id_field->offset / 8;
8491
8492         /* Maximum number of groups. */
8493         params->n_groups_max = s->n_groups_max;
8494
8495         /* Maximum number of members per group. */
8496         params->n_members_per_group_max = s->n_members_per_group_max;
8497
8498         return params;
8499
8500 error:
8501         selector_params_free(params);
8502         return NULL;
8503 }
8504
8505 static void
8506 selector_build_free(struct rte_swx_pipeline *p)
8507 {
8508         uint32_t i;
8509
8510         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8511                 struct thread *t = &p->threads[i];
8512                 uint32_t j;
8513
8514                 if (!t->selectors)
8515                         continue;
8516
8517                 for (j = 0; j < p->n_selectors; j++) {
8518                         struct selector_runtime *r = &t->selectors[j];
8519
8520                         free(r->mailbox);
8521                 }
8522
8523                 free(t->selectors);
8524                 t->selectors = NULL;
8525         }
8526
8527         free(p->selector_stats);
8528         p->selector_stats = NULL;
8529 }
8530
8531 static int
8532 selector_build(struct rte_swx_pipeline *p)
8533 {
8534         uint32_t i;
8535         int status = 0;
8536
8537         /* Per pipeline: selector statistics. */
8538         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8539         if (!p->selector_stats) {
8540                 status = -ENOMEM;
8541                 goto error;
8542         }
8543
8544         /* Per thread: selector run-time. */
8545         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8546                 struct thread *t = &p->threads[i];
8547                 struct selector *s;
8548
8549                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8550                 if (!t->selectors) {
8551                         status = -ENOMEM;
8552                         goto error;
8553                 }
8554
8555                 TAILQ_FOREACH(s, &p->selectors, node) {
8556                         struct selector_runtime *r = &t->selectors[s->id];
8557                         uint64_t size;
8558
8559                         /* r->mailbox. */
8560                         size = rte_swx_table_selector_mailbox_size_get();
8561                         if (size) {
8562                                 r->mailbox = calloc(1, size);
8563                                 if (!r->mailbox) {
8564                                         status = -ENOMEM;
8565                                         goto error;
8566                                 }
8567                         }
8568
8569                         /* r->group_id_buffer. */
8570                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8571
8572                         /* r->selector_buffer. */
8573                         r->selector_buffer = s->selector_header ?
8574                                 &t->structs[s->selector_header->struct_id] :
8575                                 &t->structs[p->metadata_struct_id];
8576
8577                         /* r->member_id_buffer. */
8578                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8579                 }
8580         }
8581
8582         return 0;
8583
8584 error:
8585         selector_build_free(p);
8586         return status;
8587 }
8588
8589 static void
8590 selector_free(struct rte_swx_pipeline *p)
8591 {
8592         selector_build_free(p);
8593
8594         /* Selector tables. */
8595         for ( ; ; ) {
8596                 struct selector *elem;
8597
8598                 elem = TAILQ_FIRST(&p->selectors);
8599                 if (!elem)
8600                         break;
8601
8602                 TAILQ_REMOVE(&p->selectors, elem, node);
8603                 free(elem->selector_fields);
8604                 free(elem);
8605         }
8606 }
8607
8608 /*
8609  * Learner table.
8610  */
8611 static struct learner *
8612 learner_find(struct rte_swx_pipeline *p, const char *name)
8613 {
8614         struct learner *l;
8615
8616         TAILQ_FOREACH(l, &p->learners, node)
8617                 if (!strcmp(l->name, name))
8618                         return l;
8619
8620         return NULL;
8621 }
8622
8623 static struct learner *
8624 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8625 {
8626         struct learner *l = NULL;
8627
8628         TAILQ_FOREACH(l, &p->learners, node)
8629                 if (l->id == id)
8630                         return l;
8631
8632         return NULL;
8633 }
8634
8635 static int
8636 learner_match_fields_check(struct rte_swx_pipeline *p,
8637                            struct rte_swx_pipeline_learner_params *params,
8638                            struct header **header)
8639 {
8640         struct header *h0 = NULL;
8641         struct field *hf, *mf;
8642         uint32_t i;
8643
8644         /* Return if no match fields. */
8645         if (!params->n_fields || !params->field_names)
8646                 return -EINVAL;
8647
8648         /* Check that all the match fields either belong to the same header
8649          * or are all meta-data fields.
8650          */
8651         hf = header_field_parse(p, params->field_names[0], &h0);
8652         mf = metadata_field_parse(p, params->field_names[0]);
8653         if (!hf && !mf)
8654                 return -EINVAL;
8655
8656         for (i = 1; i < params->n_fields; i++)
8657                 if (h0) {
8658                         struct header *h;
8659
8660                         hf = header_field_parse(p, params->field_names[i], &h);
8661                         if (!hf || (h->id != h0->id))
8662                                 return -EINVAL;
8663                 } else {
8664                         mf = metadata_field_parse(p, params->field_names[i]);
8665                         if (!mf)
8666                                 return -EINVAL;
8667                 }
8668
8669         /* Check that there are no duplicated match fields. */
8670         for (i = 0; i < params->n_fields; i++) {
8671                 const char *field_name = params->field_names[i];
8672                 uint32_t j;
8673
8674                 for (j = i + 1; j < params->n_fields; j++)
8675                         if (!strcmp(params->field_names[j], field_name))
8676                                 return -EINVAL;
8677         }
8678
8679         /* Return. */
8680         if (header)
8681                 *header = h0;
8682
8683         return 0;
8684 }
8685
8686 static int
8687 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8688 {
8689         struct struct_type *mst = p->metadata_st, *ast = a->st;
8690         struct field *mf, *af;
8691         uint32_t mf_pos, i;
8692
8693         if (!ast) {
8694                 if (mf_name)
8695                         return -EINVAL;
8696
8697                 return 0;
8698         }
8699
8700         /* Check that mf_name is the name of a valid meta-data field. */
8701         CHECK_NAME(mf_name, EINVAL);
8702         mf = metadata_field_parse(p, mf_name);
8703         CHECK(mf, EINVAL);
8704
8705         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8706          * all the action arguments.
8707          */
8708         mf_pos = mf - mst->fields;
8709         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8710
8711         /* Check that the size of each of the identified meta-data fields matches exactly the size
8712          * of the corresponding action argument.
8713          */
8714         for (i = 0; i < ast->n_fields; i++) {
8715                 mf = &mst->fields[mf_pos + i];
8716                 af = &ast->fields[i];
8717
8718                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8719         }
8720
8721         return 0;
8722 }
8723
8724 static int
8725 learner_action_learning_check(struct rte_swx_pipeline *p,
8726                               struct action *action,
8727                               const char **action_names,
8728                               uint32_t n_actions)
8729 {
8730         uint32_t i;
8731
8732         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8733          * the action passed as argument to the "learn" instruction) is also enabled for the
8734          * current learner table.
8735          */
8736         for (i = 0; i < action->n_instructions; i++) {
8737                 struct instruction *instr = &action->instructions[i];
8738                 uint32_t found = 0, j;
8739
8740                 if (instr->type != INSTR_LEARNER_LEARN)
8741                         continue;
8742
8743                 for (j = 0; j < n_actions; j++) {
8744                         struct action *a;
8745
8746                         a = action_find(p, action_names[j]);
8747                         if (!a)
8748                                 return -EINVAL;
8749
8750                         if (a->id == instr->learn.action_id)
8751                                 found = 1;
8752                 }
8753
8754                 if (!found)
8755                         return -EINVAL;
8756         }
8757
8758         return 0;
8759 }
8760
8761 int
8762 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8763                               const char *name,
8764                               struct rte_swx_pipeline_learner_params *params,
8765                               uint32_t size,
8766                               uint32_t *timeout,
8767                               uint32_t n_timeouts)
8768 {
8769         struct learner *l = NULL;
8770         struct action *default_action;
8771         struct header *header = NULL;
8772         uint32_t action_data_size_max = 0, i;
8773         int status = 0;
8774
8775         CHECK(p, EINVAL);
8776
8777         CHECK_NAME(name, EINVAL);
8778         CHECK(!table_find(p, name), EEXIST);
8779         CHECK(!selector_find(p, name), EEXIST);
8780         CHECK(!learner_find(p, name), EEXIST);
8781
8782         CHECK(params, EINVAL);
8783
8784         /* Match checks. */
8785         status = learner_match_fields_check(p, params, &header);
8786         if (status)
8787                 return status;
8788
8789         /* Action checks. */
8790         CHECK(params->n_actions, EINVAL);
8791         CHECK(params->action_names, EINVAL);
8792         for (i = 0; i < params->n_actions; i++) {
8793                 const char *action_name = params->action_names[i];
8794                 struct action *a;
8795                 uint32_t action_data_size;
8796                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8797
8798                 CHECK_NAME(action_name, EINVAL);
8799
8800                 a = action_find(p, action_name);
8801                 CHECK(a, EINVAL);
8802
8803                 status = learner_action_learning_check(p,
8804                                                        a,
8805                                                        params->action_names,
8806                                                        params->n_actions);
8807                 if (status)
8808                         return status;
8809
8810                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8811                 if (action_data_size > action_data_size_max)
8812                         action_data_size_max = action_data_size;
8813
8814                 if (params->action_is_for_table_entries)
8815                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8816                 if (params->action_is_for_default_entry)
8817                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8818                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8819         }
8820
8821         CHECK_NAME(params->default_action_name, EINVAL);
8822         for (i = 0; i < p->n_actions; i++)
8823                 if (!strcmp(params->action_names[i],
8824                             params->default_action_name))
8825                         break;
8826         CHECK(i < params->n_actions, EINVAL);
8827         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8828               EINVAL);
8829
8830         default_action = action_find(p, params->default_action_name);
8831         CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
8832               EINVAL);
8833
8834         /* Any other checks. */
8835         CHECK(size, EINVAL);
8836         CHECK(timeout, EINVAL);
8837         CHECK(n_timeouts && (n_timeouts <= RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX), EINVAL);
8838
8839         /* Memory allocation. */
8840         l = calloc(1, sizeof(struct learner));
8841         if (!l) {
8842                 status = -ENOMEM;
8843                 goto error;
8844         }
8845
8846         l->fields = calloc(params->n_fields, sizeof(struct field *));
8847         if (!l->fields) {
8848                 status = -ENOMEM;
8849                 goto error;
8850         }
8851
8852         l->actions = calloc(params->n_actions, sizeof(struct action *));
8853         if (!l->actions) {
8854                 status = -ENOMEM;
8855                 goto error;
8856         }
8857
8858         if (action_data_size_max) {
8859                 l->default_action_data = calloc(1, action_data_size_max);
8860                 if (!l->default_action_data) {
8861                         status = -ENOMEM;
8862                         goto error;
8863                 }
8864         }
8865
8866         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8867         if (!l->action_is_for_table_entries) {
8868                 status = -ENOMEM;
8869                 goto error;
8870         }
8871
8872         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8873         if (!l->action_is_for_default_entry) {
8874                 status = -ENOMEM;
8875                 goto error;
8876         }
8877
8878         /* Node initialization. */
8879         strcpy(l->name, name);
8880
8881         for (i = 0; i < params->n_fields; i++) {
8882                 const char *field_name = params->field_names[i];
8883
8884                 l->fields[i] = header ?
8885                         header_field_parse(p, field_name, NULL) :
8886                         metadata_field_parse(p, field_name);
8887         }
8888
8889         l->n_fields = params->n_fields;
8890
8891         l->header = header;
8892
8893         for (i = 0; i < params->n_actions; i++) {
8894                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8895
8896                 if (params->action_is_for_table_entries)
8897                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8898                 if (params->action_is_for_default_entry)
8899                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8900
8901                 l->actions[i] = action_find(p, params->action_names[i]);
8902                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8903                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8904         }
8905
8906         l->default_action = default_action;
8907
8908         if (default_action->st) {
8909                 status = action_args_parse(default_action,
8910                                            params->default_action_args,
8911                                            l->default_action_data);
8912                 if (status)
8913                         goto error;
8914         }
8915
8916         l->n_actions = params->n_actions;
8917
8918         l->default_action_is_const = params->default_action_is_const;
8919
8920         l->action_data_size_max = action_data_size_max;
8921
8922         l->size = size;
8923
8924         for (i = 0; i < n_timeouts; i++)
8925                 l->timeout[i] = timeout[i];
8926
8927         l->n_timeouts = n_timeouts;
8928
8929         l->id = p->n_learners;
8930
8931         /* Node add to tailq. */
8932         TAILQ_INSERT_TAIL(&p->learners, l, node);
8933         p->n_learners++;
8934
8935         return 0;
8936
8937 error:
8938         if (!l)
8939                 return status;
8940
8941         free(l->action_is_for_default_entry);
8942         free(l->action_is_for_table_entries);
8943         free(l->default_action_data);
8944         free(l->actions);
8945         free(l->fields);
8946         free(l);
8947
8948         return status;
8949 }
8950
8951 static void
8952 learner_params_free(struct rte_swx_table_learner_params *params)
8953 {
8954         if (!params)
8955                 return;
8956
8957         free(params->key_mask0);
8958
8959         free(params->key_timeout);
8960
8961         free(params);
8962 }
8963
8964 static struct rte_swx_table_learner_params *
8965 learner_params_get(struct learner *l)
8966 {
8967         struct rte_swx_table_learner_params *params = NULL;
8968         struct field *first, *last;
8969         uint32_t i;
8970
8971         /* Memory allocation. */
8972         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8973         if (!params)
8974                 goto error;
8975
8976         /* Find first (smallest offset) and last (biggest offset) match fields. */
8977         first = l->fields[0];
8978         last = l->fields[0];
8979
8980         for (i = 0; i < l->n_fields; i++) {
8981                 struct field *f = l->fields[i];
8982
8983                 if (f->offset < first->offset)
8984                         first = f;
8985
8986                 if (f->offset > last->offset)
8987                         last = f;
8988         }
8989
8990         /* Key offset and size. */
8991         params->key_offset = first->offset / 8;
8992         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8993
8994         /* Memory allocation. */
8995         params->key_mask0 = calloc(1, params->key_size);
8996         if (!params->key_mask0)
8997                 goto error;
8998
8999         /* Key mask. */
9000         for (i = 0; i < l->n_fields; i++) {
9001                 struct field *f = l->fields[i];
9002                 uint32_t start = (f->offset - first->offset) / 8;
9003                 size_t size = f->n_bits / 8;
9004
9005                 memset(&params->key_mask0[start], 0xFF, size);
9006         }
9007
9008         /* Action data size. */
9009         params->action_data_size = l->action_data_size_max;
9010
9011         /* Maximum number of keys. */
9012         params->n_keys_max = l->size;
9013
9014         /* Memory allocation. */
9015         params->key_timeout = calloc(l->n_timeouts, sizeof(uint32_t));
9016         if (!params->key_timeout)
9017                 goto error;
9018
9019         /* Timeout. */
9020         for (i = 0; i < l->n_timeouts; i++)
9021                 params->key_timeout[i] = l->timeout[i];
9022
9023         params->n_key_timeouts = l->n_timeouts;
9024
9025         return params;
9026
9027 error:
9028         learner_params_free(params);
9029         return NULL;
9030 }
9031
9032 static void
9033 learner_build_free(struct rte_swx_pipeline *p)
9034 {
9035         uint32_t i;
9036
9037         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9038                 struct thread *t = &p->threads[i];
9039                 uint32_t j;
9040
9041                 if (!t->learners)
9042                         continue;
9043
9044                 for (j = 0; j < p->n_learners; j++) {
9045                         struct learner_runtime *r = &t->learners[j];
9046
9047                         free(r->mailbox);
9048                 }
9049
9050                 free(t->learners);
9051                 t->learners = NULL;
9052         }
9053
9054         if (p->learner_stats) {
9055                 for (i = 0; i < p->n_learners; i++)
9056                         free(p->learner_stats[i].n_pkts_action);
9057
9058                 free(p->learner_stats);
9059         }
9060 }
9061
9062 static int
9063 learner_build(struct rte_swx_pipeline *p)
9064 {
9065         uint32_t i;
9066         int status = 0;
9067
9068         /* Per pipeline: learner statistics. */
9069         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9070         CHECK(p->learner_stats, ENOMEM);
9071
9072         for (i = 0; i < p->n_learners; i++) {
9073                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9074                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9075         }
9076
9077         /* Per thread: learner run-time. */
9078         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9079                 struct thread *t = &p->threads[i];
9080                 struct learner *l;
9081
9082                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9083                 if (!t->learners) {
9084                         status = -ENOMEM;
9085                         goto error;
9086                 }
9087
9088                 TAILQ_FOREACH(l, &p->learners, node) {
9089                         struct learner_runtime *r = &t->learners[l->id];
9090                         uint64_t size;
9091
9092                         /* r->mailbox. */
9093                         size = rte_swx_table_learner_mailbox_size_get();
9094                         if (size) {
9095                                 r->mailbox = calloc(1, size);
9096                                 if (!r->mailbox) {
9097                                         status = -ENOMEM;
9098                                         goto error;
9099                                 }
9100                         }
9101
9102                         /* r->key. */
9103                         r->key = l->header ?
9104                                 &t->structs[l->header->struct_id] :
9105                                 &t->structs[p->metadata_struct_id];
9106                 }
9107         }
9108
9109         return 0;
9110
9111 error:
9112         learner_build_free(p);
9113         return status;
9114 }
9115
9116 static void
9117 learner_free(struct rte_swx_pipeline *p)
9118 {
9119         learner_build_free(p);
9120
9121         /* Learner tables. */
9122         for ( ; ; ) {
9123                 struct learner *l;
9124
9125                 l = TAILQ_FIRST(&p->learners);
9126                 if (!l)
9127                         break;
9128
9129                 TAILQ_REMOVE(&p->learners, l, node);
9130                 free(l->fields);
9131                 free(l->actions);
9132                 free(l->default_action_data);
9133                 free(l);
9134         }
9135 }
9136
9137 /*
9138  * Table state.
9139  */
9140 static int
9141 table_state_build(struct rte_swx_pipeline *p)
9142 {
9143         struct table *table;
9144         struct selector *s;
9145         struct learner *l;
9146
9147         p->table_state = calloc(p->n_tables + p->n_selectors + p->n_learners,
9148                                 sizeof(struct rte_swx_table_state));
9149         CHECK(p->table_state, ENOMEM);
9150
9151         TAILQ_FOREACH(table, &p->tables, node) {
9152                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9153
9154                 if (table->type) {
9155                         struct rte_swx_table_params *params;
9156
9157                         /* ts->obj. */
9158                         params = table_params_get(table);
9159                         CHECK(params, ENOMEM);
9160
9161                         ts->obj = table->type->ops.create(params,
9162                                 NULL,
9163                                 table->args,
9164                                 p->numa_node);
9165
9166                         table_params_free(params);
9167                         CHECK(ts->obj, ENODEV);
9168                 }
9169
9170                 /* ts->default_action_data. */
9171                 if (table->action_data_size_max) {
9172                         ts->default_action_data =
9173                                 malloc(table->action_data_size_max);
9174                         CHECK(ts->default_action_data, ENOMEM);
9175
9176                         memcpy(ts->default_action_data,
9177                                table->default_action_data,
9178                                table->action_data_size_max);
9179                 }
9180
9181                 /* ts->default_action_id. */
9182                 ts->default_action_id = table->default_action->id;
9183         }
9184
9185         TAILQ_FOREACH(s, &p->selectors, node) {
9186                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9187                 struct rte_swx_table_selector_params *params;
9188
9189                 /* ts->obj. */
9190                 params = selector_table_params_get(s);
9191                 CHECK(params, ENOMEM);
9192
9193                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9194
9195                 selector_params_free(params);
9196                 CHECK(ts->obj, ENODEV);
9197         }
9198
9199         TAILQ_FOREACH(l, &p->learners, node) {
9200                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9201                         p->n_selectors + l->id];
9202                 struct rte_swx_table_learner_params *params;
9203
9204                 /* ts->obj. */
9205                 params = learner_params_get(l);
9206                 CHECK(params, ENOMEM);
9207
9208                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9209                 learner_params_free(params);
9210                 CHECK(ts->obj, ENODEV);
9211
9212                 /* ts->default_action_data. */
9213                 if (l->action_data_size_max) {
9214                         ts->default_action_data = malloc(l->action_data_size_max);
9215                         CHECK(ts->default_action_data, ENOMEM);
9216
9217                         memcpy(ts->default_action_data,
9218                                l->default_action_data,
9219                                l->action_data_size_max);
9220                 }
9221
9222                 /* ts->default_action_id. */
9223                 ts->default_action_id = l->default_action->id;
9224         }
9225
9226         return 0;
9227 }
9228
9229 static void
9230 table_state_build_free(struct rte_swx_pipeline *p)
9231 {
9232         uint32_t i;
9233
9234         if (!p->table_state)
9235                 return;
9236
9237         for (i = 0; i < p->n_tables; i++) {
9238                 struct rte_swx_table_state *ts = &p->table_state[i];
9239                 struct table *table = table_find_by_id(p, i);
9240
9241                 /* ts->obj. */
9242                 if (table->type && ts->obj)
9243                         table->type->ops.free(ts->obj);
9244
9245                 /* ts->default_action_data. */
9246                 free(ts->default_action_data);
9247         }
9248
9249         for (i = 0; i < p->n_selectors; i++) {
9250                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9251
9252                 /* ts->obj. */
9253                 if (ts->obj)
9254                         rte_swx_table_selector_free(ts->obj);
9255         }
9256
9257         for (i = 0; i < p->n_learners; i++) {
9258                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9259
9260                 /* ts->obj. */
9261                 if (ts->obj)
9262                         rte_swx_table_learner_free(ts->obj);
9263
9264                 /* ts->default_action_data. */
9265                 free(ts->default_action_data);
9266         }
9267
9268         free(p->table_state);
9269         p->table_state = NULL;
9270 }
9271
9272 static void
9273 table_state_free(struct rte_swx_pipeline *p)
9274 {
9275         table_state_build_free(p);
9276 }
9277
9278 /*
9279  * Register array.
9280  */
9281 static struct regarray *
9282 regarray_find(struct rte_swx_pipeline *p, const char *name)
9283 {
9284         struct regarray *elem;
9285
9286         TAILQ_FOREACH(elem, &p->regarrays, node)
9287                 if (!strcmp(elem->name, name))
9288                         return elem;
9289
9290         return NULL;
9291 }
9292
9293 static struct regarray *
9294 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9295 {
9296         struct regarray *elem = NULL;
9297
9298         TAILQ_FOREACH(elem, &p->regarrays, node)
9299                 if (elem->id == id)
9300                         return elem;
9301
9302         return NULL;
9303 }
9304
9305 int
9306 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9307                               const char *name,
9308                               uint32_t size,
9309                               uint64_t init_val)
9310 {
9311         struct regarray *r;
9312
9313         CHECK(p, EINVAL);
9314
9315         CHECK_NAME(name, EINVAL);
9316         CHECK(!regarray_find(p, name), EEXIST);
9317
9318         CHECK(size, EINVAL);
9319         size = rte_align32pow2(size);
9320
9321         /* Memory allocation. */
9322         r = calloc(1, sizeof(struct regarray));
9323         CHECK(r, ENOMEM);
9324
9325         /* Node initialization. */
9326         strcpy(r->name, name);
9327         r->init_val = init_val;
9328         r->size = size;
9329         r->id = p->n_regarrays;
9330
9331         /* Node add to tailq. */
9332         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9333         p->n_regarrays++;
9334
9335         return 0;
9336 }
9337
9338 static int
9339 regarray_build(struct rte_swx_pipeline *p)
9340 {
9341         struct regarray *regarray;
9342
9343         if (!p->n_regarrays)
9344                 return 0;
9345
9346         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9347         CHECK(p->regarray_runtime, ENOMEM);
9348
9349         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9350                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9351                 uint32_t i;
9352
9353                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9354                                          RTE_CACHE_LINE_SIZE,
9355                                          p->numa_node);
9356                 CHECK(r->regarray, ENOMEM);
9357
9358                 if (regarray->init_val)
9359                         for (i = 0; i < regarray->size; i++)
9360                                 r->regarray[i] = regarray->init_val;
9361
9362                 r->size_mask = regarray->size - 1;
9363         }
9364
9365         return 0;
9366 }
9367
9368 static void
9369 regarray_build_free(struct rte_swx_pipeline *p)
9370 {
9371         uint32_t i;
9372
9373         if (!p->regarray_runtime)
9374                 return;
9375
9376         for (i = 0; i < p->n_regarrays; i++) {
9377                 struct regarray *regarray = regarray_find_by_id(p, i);
9378                 struct regarray_runtime *r = &p->regarray_runtime[i];
9379
9380                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9381         }
9382
9383         free(p->regarray_runtime);
9384         p->regarray_runtime = NULL;
9385 }
9386
9387 static void
9388 regarray_free(struct rte_swx_pipeline *p)
9389 {
9390         regarray_build_free(p);
9391
9392         for ( ; ; ) {
9393                 struct regarray *elem;
9394
9395                 elem = TAILQ_FIRST(&p->regarrays);
9396                 if (!elem)
9397                         break;
9398
9399                 TAILQ_REMOVE(&p->regarrays, elem, node);
9400                 free(elem);
9401         }
9402 }
9403
9404 /*
9405  * Meter array.
9406  */
9407 static struct meter_profile *
9408 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9409 {
9410         struct meter_profile *elem;
9411
9412         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9413                 if (!strcmp(elem->name, name))
9414                         return elem;
9415
9416         return NULL;
9417 }
9418
9419 static struct metarray *
9420 metarray_find(struct rte_swx_pipeline *p, const char *name)
9421 {
9422         struct metarray *elem;
9423
9424         TAILQ_FOREACH(elem, &p->metarrays, node)
9425                 if (!strcmp(elem->name, name))
9426                         return elem;
9427
9428         return NULL;
9429 }
9430
9431 static struct metarray *
9432 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9433 {
9434         struct metarray *elem = NULL;
9435
9436         TAILQ_FOREACH(elem, &p->metarrays, node)
9437                 if (elem->id == id)
9438                         return elem;
9439
9440         return NULL;
9441 }
9442
9443 int
9444 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9445                                  const char *name,
9446                                  uint32_t size)
9447 {
9448         struct metarray *m;
9449
9450         CHECK(p, EINVAL);
9451
9452         CHECK_NAME(name, EINVAL);
9453         CHECK(!metarray_find(p, name), EEXIST);
9454
9455         CHECK(size, EINVAL);
9456         size = rte_align32pow2(size);
9457
9458         /* Memory allocation. */
9459         m = calloc(1, sizeof(struct metarray));
9460         CHECK(m, ENOMEM);
9461
9462         /* Node initialization. */
9463         strcpy(m->name, name);
9464         m->size = size;
9465         m->id = p->n_metarrays;
9466
9467         /* Node add to tailq. */
9468         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9469         p->n_metarrays++;
9470
9471         return 0;
9472 }
9473
9474 struct meter_profile meter_profile_default = {
9475         .node = {0},
9476         .name = "",
9477         .params = {0},
9478
9479         .profile = {
9480                 .cbs = 10000,
9481                 .pbs = 10000,
9482                 .cir_period = 1,
9483                 .cir_bytes_per_period = 1,
9484                 .pir_period = 1,
9485                 .pir_bytes_per_period = 1,
9486         },
9487
9488         .n_users = 0,
9489 };
9490
9491 static void
9492 meter_init(struct meter *m)
9493 {
9494         memset(m, 0, sizeof(struct meter));
9495         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9496         m->profile = &meter_profile_default;
9497         m->color_mask = RTE_COLOR_GREEN;
9498
9499         meter_profile_default.n_users++;
9500 }
9501
9502 static int
9503 metarray_build(struct rte_swx_pipeline *p)
9504 {
9505         struct metarray *m;
9506
9507         if (!p->n_metarrays)
9508                 return 0;
9509
9510         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9511         CHECK(p->metarray_runtime, ENOMEM);
9512
9513         TAILQ_FOREACH(m, &p->metarrays, node) {
9514                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9515                 uint32_t i;
9516
9517                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9518                                          RTE_CACHE_LINE_SIZE,
9519                                          p->numa_node);
9520                 CHECK(r->metarray, ENOMEM);
9521
9522                 for (i = 0; i < m->size; i++)
9523                         meter_init(&r->metarray[i]);
9524
9525                 r->size_mask = m->size - 1;
9526         }
9527
9528         return 0;
9529 }
9530
9531 static void
9532 metarray_build_free(struct rte_swx_pipeline *p)
9533 {
9534         uint32_t i;
9535
9536         if (!p->metarray_runtime)
9537                 return;
9538
9539         for (i = 0; i < p->n_metarrays; i++) {
9540                 struct metarray *m = metarray_find_by_id(p, i);
9541                 struct metarray_runtime *r = &p->metarray_runtime[i];
9542
9543                 env_free(r->metarray, m->size * sizeof(struct meter));
9544         }
9545
9546         free(p->metarray_runtime);
9547         p->metarray_runtime = NULL;
9548 }
9549
9550 static void
9551 metarray_free(struct rte_swx_pipeline *p)
9552 {
9553         metarray_build_free(p);
9554
9555         /* Meter arrays. */
9556         for ( ; ; ) {
9557                 struct metarray *elem;
9558
9559                 elem = TAILQ_FIRST(&p->metarrays);
9560                 if (!elem)
9561                         break;
9562
9563                 TAILQ_REMOVE(&p->metarrays, elem, node);
9564                 free(elem);
9565         }
9566
9567         /* Meter profiles. */
9568         for ( ; ; ) {
9569                 struct meter_profile *elem;
9570
9571                 elem = TAILQ_FIRST(&p->meter_profiles);
9572                 if (!elem)
9573                         break;
9574
9575                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9576                 free(elem);
9577         }
9578 }
9579
9580 /*
9581  * Pipeline.
9582  */
9583 void
9584 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9585 {
9586         void *lib;
9587
9588         if (!p)
9589                 return;
9590
9591         lib = p->lib;
9592
9593         free(p->instruction_data);
9594         free(p->instructions);
9595
9596         metarray_free(p);
9597         regarray_free(p);
9598         table_state_free(p);
9599         learner_free(p);
9600         selector_free(p);
9601         table_free(p);
9602         action_free(p);
9603         instruction_table_free(p);
9604         metadata_free(p);
9605         header_free(p);
9606         hash_func_free(p);
9607         extern_func_free(p);
9608         extern_obj_free(p);
9609         mirroring_free(p);
9610         port_out_free(p);
9611         port_in_free(p);
9612         struct_free(p);
9613
9614         free(p);
9615
9616         if (lib)
9617                 dlclose(lib);
9618 }
9619
9620 static int
9621 port_in_types_register(struct rte_swx_pipeline *p)
9622 {
9623         int status;
9624
9625         status = rte_swx_pipeline_port_in_type_register(p,
9626                 "ethdev",
9627                 &rte_swx_port_ethdev_reader_ops);
9628         if (status)
9629                 return status;
9630
9631         status = rte_swx_pipeline_port_in_type_register(p,
9632                 "ring",
9633                 &rte_swx_port_ring_reader_ops);
9634         if (status)
9635                 return status;
9636
9637 #ifdef RTE_PORT_PCAP
9638         status = rte_swx_pipeline_port_in_type_register(p,
9639                 "source",
9640                 &rte_swx_port_source_ops);
9641         if (status)
9642                 return status;
9643 #endif
9644
9645         status = rte_swx_pipeline_port_in_type_register(p,
9646                 "fd",
9647                 &rte_swx_port_fd_reader_ops);
9648         if (status)
9649                 return status;
9650
9651         return 0;
9652 }
9653
9654 static int
9655 port_out_types_register(struct rte_swx_pipeline *p)
9656 {
9657         int status;
9658
9659         status = rte_swx_pipeline_port_out_type_register(p,
9660                 "ethdev",
9661                 &rte_swx_port_ethdev_writer_ops);
9662         if (status)
9663                 return status;
9664
9665         status = rte_swx_pipeline_port_out_type_register(p,
9666                 "ring",
9667                 &rte_swx_port_ring_writer_ops);
9668         if (status)
9669                 return status;
9670
9671         status = rte_swx_pipeline_port_out_type_register(p,
9672                 "sink",
9673                 &rte_swx_port_sink_ops);
9674         if (status)
9675                 return status;
9676
9677         status = rte_swx_pipeline_port_out_type_register(p,
9678                 "fd",
9679                 &rte_swx_port_fd_writer_ops);
9680         if (status)
9681                 return status;
9682
9683         return 0;
9684 }
9685
9686 static int
9687 table_types_register(struct rte_swx_pipeline *p)
9688 {
9689         int status;
9690
9691         status = rte_swx_pipeline_table_type_register(p,
9692                 "exact",
9693                 RTE_SWX_TABLE_MATCH_EXACT,
9694                 &rte_swx_table_exact_match_ops);
9695         if (status)
9696                 return status;
9697
9698         status = rte_swx_pipeline_table_type_register(p,
9699                 "wildcard",
9700                 RTE_SWX_TABLE_MATCH_WILDCARD,
9701                 &rte_swx_table_wildcard_match_ops);
9702         if (status)
9703                 return status;
9704
9705         return 0;
9706 }
9707
9708 static int
9709 hash_funcs_register(struct rte_swx_pipeline *p)
9710 {
9711         int status;
9712
9713         status = rte_swx_pipeline_hash_func_register(p, "jhash", rte_jhash);
9714         if (status)
9715                 return status;
9716
9717         status = rte_swx_pipeline_hash_func_register(p, "crc32", rte_hash_crc);
9718         if (status)
9719                 return status;
9720
9721         return 0;
9722 }
9723
9724 int
9725 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9726 {
9727         struct rte_swx_pipeline *pipeline = NULL;
9728         int status = 0;
9729
9730         /* Check input parameters. */
9731         CHECK(p, EINVAL);
9732
9733         /* Memory allocation. */
9734         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9735         if (!pipeline) {
9736                 status = -ENOMEM;
9737                 goto error;
9738         }
9739
9740         /* Initialization. */
9741         TAILQ_INIT(&pipeline->struct_types);
9742         TAILQ_INIT(&pipeline->port_in_types);
9743         TAILQ_INIT(&pipeline->ports_in);
9744         TAILQ_INIT(&pipeline->port_out_types);
9745         TAILQ_INIT(&pipeline->ports_out);
9746         TAILQ_INIT(&pipeline->extern_types);
9747         TAILQ_INIT(&pipeline->extern_objs);
9748         TAILQ_INIT(&pipeline->extern_funcs);
9749         TAILQ_INIT(&pipeline->hash_funcs);
9750         TAILQ_INIT(&pipeline->headers);
9751         TAILQ_INIT(&pipeline->actions);
9752         TAILQ_INIT(&pipeline->table_types);
9753         TAILQ_INIT(&pipeline->tables);
9754         TAILQ_INIT(&pipeline->selectors);
9755         TAILQ_INIT(&pipeline->learners);
9756         TAILQ_INIT(&pipeline->regarrays);
9757         TAILQ_INIT(&pipeline->meter_profiles);
9758         TAILQ_INIT(&pipeline->metarrays);
9759
9760         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9761         pipeline->n_mirroring_slots = RTE_SWX_PACKET_MIRRORING_SLOTS_DEFAULT;
9762         pipeline->n_mirroring_sessions = RTE_SWX_PACKET_MIRRORING_SESSIONS_DEFAULT;
9763         pipeline->numa_node = numa_node;
9764
9765         status = port_in_types_register(pipeline);
9766         if (status)
9767                 goto error;
9768
9769         status = port_out_types_register(pipeline);
9770         if (status)
9771                 goto error;
9772
9773         status = table_types_register(pipeline);
9774         if (status)
9775                 goto error;
9776
9777         status = hash_funcs_register(pipeline);
9778         if (status)
9779                 goto error;
9780
9781         *p = pipeline;
9782         return 0;
9783
9784 error:
9785         rte_swx_pipeline_free(pipeline);
9786         return status;
9787 }
9788
9789 int
9790 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9791                                      const char **instructions,
9792                                      uint32_t n_instructions)
9793 {
9794         int err;
9795         uint32_t i;
9796
9797         err = instruction_config(p, NULL, instructions, n_instructions);
9798         if (err)
9799                 return err;
9800
9801         /* Thread instruction pointer reset. */
9802         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9803                 struct thread *t = &p->threads[i];
9804
9805                 thread_ip_reset(p, t);
9806         }
9807
9808         return 0;
9809 }
9810
9811 static int
9812 pipeline_compile(struct rte_swx_pipeline *p);
9813
9814 int
9815 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9816 {
9817         struct rte_swx_port_sink_params drop_port_params = {
9818                 .file_name = NULL,
9819         };
9820         int status;
9821
9822         CHECK(p, EINVAL);
9823         CHECK(p->build_done == 0, EEXIST);
9824
9825         status = port_in_build(p);
9826         if (status)
9827                 goto error;
9828
9829         /* Drop port. */
9830         status = rte_swx_pipeline_port_out_config(p,
9831                                                   p->n_ports_out,
9832                                                   "sink",
9833                                                   &drop_port_params);
9834         if (status)
9835                 goto error;
9836
9837         status = port_out_build(p);
9838         if (status)
9839                 goto error;
9840
9841         status = mirroring_build(p);
9842         if (status)
9843                 goto error;
9844
9845         status = struct_build(p);
9846         if (status)
9847                 goto error;
9848
9849         status = extern_obj_build(p);
9850         if (status)
9851                 goto error;
9852
9853         status = extern_func_build(p);
9854         if (status)
9855                 goto error;
9856
9857         status = hash_func_build(p);
9858         if (status)
9859                 goto error;
9860
9861         status = header_build(p);
9862         if (status)
9863                 goto error;
9864
9865         status = metadata_build(p);
9866         if (status)
9867                 goto error;
9868
9869         status = instruction_table_build(p);
9870         if (status)
9871                 goto error;
9872
9873         status = action_build(p);
9874         if (status)
9875                 goto error;
9876
9877         status = table_build(p);
9878         if (status)
9879                 goto error;
9880
9881         status = selector_build(p);
9882         if (status)
9883                 goto error;
9884
9885         status = learner_build(p);
9886         if (status)
9887                 goto error;
9888
9889         status = table_state_build(p);
9890         if (status)
9891                 goto error;
9892
9893         status = regarray_build(p);
9894         if (status)
9895                 goto error;
9896
9897         status = metarray_build(p);
9898         if (status)
9899                 goto error;
9900
9901         p->build_done = 1;
9902
9903         pipeline_compile(p);
9904
9905         return 0;
9906
9907 error:
9908         metarray_build_free(p);
9909         regarray_build_free(p);
9910         table_state_build_free(p);
9911         learner_build_free(p);
9912         selector_build_free(p);
9913         table_build_free(p);
9914         action_build_free(p);
9915         instruction_table_build_free(p);
9916         metadata_build_free(p);
9917         header_build_free(p);
9918         hash_func_build_free(p);
9919         extern_func_build_free(p);
9920         extern_obj_build_free(p);
9921         mirroring_build_free(p);
9922         port_out_build_free(p);
9923         port_in_build_free(p);
9924         struct_build_free(p);
9925
9926         return status;
9927 }
9928
9929 void
9930 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9931 {
9932         uint32_t i;
9933
9934         for (i = 0; i < n_instructions; i++)
9935                 instr_exec(p);
9936 }
9937
9938 void
9939 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9940 {
9941         uint32_t i;
9942
9943         for (i = 0; i < p->n_ports_out; i++) {
9944                 struct port_out_runtime *port = &p->out[i];
9945
9946                 if (port->flush)
9947                         port->flush(port->obj);
9948         }
9949 }
9950
9951 /*
9952  * Control.
9953  */
9954 int
9955 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9956                               struct rte_swx_ctl_pipeline_info *pipeline)
9957 {
9958         struct action *action;
9959         struct table *table;
9960         uint32_t n_actions = 0, n_tables = 0;
9961
9962         if (!p || !pipeline)
9963                 return -EINVAL;
9964
9965         TAILQ_FOREACH(action, &p->actions, node)
9966                 n_actions++;
9967
9968         TAILQ_FOREACH(table, &p->tables, node)
9969                 n_tables++;
9970
9971         pipeline->n_ports_in = p->n_ports_in;
9972         pipeline->n_ports_out = p->n_ports_out;
9973         pipeline->n_mirroring_slots = p->n_mirroring_slots;
9974         pipeline->n_mirroring_sessions = p->n_mirroring_sessions;
9975         pipeline->n_actions = n_actions;
9976         pipeline->n_tables = n_tables;
9977         pipeline->n_selectors = p->n_selectors;
9978         pipeline->n_learners = p->n_learners;
9979         pipeline->n_regarrays = p->n_regarrays;
9980         pipeline->n_metarrays = p->n_metarrays;
9981
9982         return 0;
9983 }
9984
9985 int
9986 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9987 {
9988         if (!p || !numa_node)
9989                 return -EINVAL;
9990
9991         *numa_node = p->numa_node;
9992         return 0;
9993 }
9994
9995 int
9996 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9997                             uint32_t action_id,
9998                             struct rte_swx_ctl_action_info *action)
9999 {
10000         struct action *a = NULL;
10001
10002         if (!p || (action_id >= p->n_actions) || !action)
10003                 return -EINVAL;
10004
10005         a = action_find_by_id(p, action_id);
10006         if (!a)
10007                 return -EINVAL;
10008
10009         strcpy(action->name, a->name);
10010         action->n_args = a->st ? a->st->n_fields : 0;
10011         return 0;
10012 }
10013
10014 int
10015 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10016                                 uint32_t action_id,
10017                                 uint32_t action_arg_id,
10018                                 struct rte_swx_ctl_action_arg_info *action_arg)
10019 {
10020         struct action *a = NULL;
10021         struct field *arg = NULL;
10022
10023         if (!p || (action_id >= p->n_actions) || !action_arg)
10024                 return -EINVAL;
10025
10026         a = action_find_by_id(p, action_id);
10027         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10028                 return -EINVAL;
10029
10030         arg = &a->st->fields[action_arg_id];
10031         strcpy(action_arg->name, arg->name);
10032         action_arg->n_bits = arg->n_bits;
10033         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10034
10035         return 0;
10036 }
10037
10038 int
10039 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10040                            uint32_t table_id,
10041                            struct rte_swx_ctl_table_info *table)
10042 {
10043         struct table *t = NULL;
10044
10045         if (!p || !table)
10046                 return -EINVAL;
10047
10048         t = table_find_by_id(p, table_id);
10049         if (!t)
10050                 return -EINVAL;
10051
10052         strcpy(table->name, t->name);
10053         strcpy(table->args, t->args);
10054         table->n_match_fields = t->n_fields;
10055         table->n_actions = t->n_actions;
10056         table->default_action_is_const = t->default_action_is_const;
10057         table->size = t->size;
10058         return 0;
10059 }
10060
10061 int
10062 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10063         uint32_t table_id,
10064         uint32_t match_field_id,
10065         struct rte_swx_ctl_table_match_field_info *match_field)
10066 {
10067         struct table *t;
10068         struct match_field *f;
10069
10070         if (!p || (table_id >= p->n_tables) || !match_field)
10071                 return -EINVAL;
10072
10073         t = table_find_by_id(p, table_id);
10074         if (!t || (match_field_id >= t->n_fields))
10075                 return -EINVAL;
10076
10077         f = &t->fields[match_field_id];
10078         match_field->match_type = f->match_type;
10079         match_field->is_header = t->header ? 1 : 0;
10080         match_field->n_bits = f->field->n_bits;
10081         match_field->offset = f->field->offset;
10082
10083         return 0;
10084 }
10085
10086 int
10087 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10088         uint32_t table_id,
10089         uint32_t table_action_id,
10090         struct rte_swx_ctl_table_action_info *table_action)
10091 {
10092         struct table *t;
10093
10094         if (!p || (table_id >= p->n_tables) || !table_action)
10095                 return -EINVAL;
10096
10097         t = table_find_by_id(p, table_id);
10098         if (!t || (table_action_id >= t->n_actions))
10099                 return -EINVAL;
10100
10101         table_action->action_id = t->actions[table_action_id]->id;
10102
10103         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
10104         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
10105
10106         return 0;
10107 }
10108
10109 int
10110 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10111                           uint32_t table_id,
10112                           struct rte_swx_table_ops *table_ops,
10113                           int *is_stub)
10114 {
10115         struct table *t;
10116
10117         if (!p || (table_id >= p->n_tables))
10118                 return -EINVAL;
10119
10120         t = table_find_by_id(p, table_id);
10121         if (!t)
10122                 return -EINVAL;
10123
10124         if (t->type) {
10125                 if (table_ops)
10126                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10127                 *is_stub = 0;
10128         } else {
10129                 *is_stub = 1;
10130         }
10131
10132         return 0;
10133 }
10134
10135 int
10136 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10137                               uint32_t selector_id,
10138                               struct rte_swx_ctl_selector_info *selector)
10139 {
10140         struct selector *s = NULL;
10141
10142         if (!p || !selector)
10143                 return -EINVAL;
10144
10145         s = selector_find_by_id(p, selector_id);
10146         if (!s)
10147                 return -EINVAL;
10148
10149         strcpy(selector->name, s->name);
10150
10151         selector->n_selector_fields = s->n_selector_fields;
10152         selector->n_groups_max = s->n_groups_max;
10153         selector->n_members_per_group_max = s->n_members_per_group_max;
10154
10155         return 0;
10156 }
10157
10158 int
10159 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10160          uint32_t selector_id,
10161          struct rte_swx_ctl_table_match_field_info *field)
10162 {
10163         struct selector *s;
10164
10165         if (!p || (selector_id >= p->n_selectors) || !field)
10166                 return -EINVAL;
10167
10168         s = selector_find_by_id(p, selector_id);
10169         if (!s)
10170                 return -EINVAL;
10171
10172         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10173         field->is_header = 0;
10174         field->n_bits = s->group_id_field->n_bits;
10175         field->offset = s->group_id_field->offset;
10176
10177         return 0;
10178 }
10179
10180 int
10181 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10182          uint32_t selector_id,
10183          uint32_t selector_field_id,
10184          struct rte_swx_ctl_table_match_field_info *field)
10185 {
10186         struct selector *s;
10187         struct field *f;
10188
10189         if (!p || (selector_id >= p->n_selectors) || !field)
10190                 return -EINVAL;
10191
10192         s = selector_find_by_id(p, selector_id);
10193         if (!s || (selector_field_id >= s->n_selector_fields))
10194                 return -EINVAL;
10195
10196         f = s->selector_fields[selector_field_id];
10197         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10198         field->is_header = s->selector_header ? 1 : 0;
10199         field->n_bits = f->n_bits;
10200         field->offset = f->offset;
10201
10202         return 0;
10203 }
10204
10205 int
10206 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10207          uint32_t selector_id,
10208          struct rte_swx_ctl_table_match_field_info *field)
10209 {
10210         struct selector *s;
10211
10212         if (!p || (selector_id >= p->n_selectors) || !field)
10213                 return -EINVAL;
10214
10215         s = selector_find_by_id(p, selector_id);
10216         if (!s)
10217                 return -EINVAL;
10218
10219         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10220         field->is_header = 0;
10221         field->n_bits = s->member_id_field->n_bits;
10222         field->offset = s->member_id_field->offset;
10223
10224         return 0;
10225 }
10226
10227 int
10228 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10229                              uint32_t learner_id,
10230                              struct rte_swx_ctl_learner_info *learner)
10231 {
10232         struct learner *l = NULL;
10233
10234         if (!p || !learner)
10235                 return -EINVAL;
10236
10237         l = learner_find_by_id(p, learner_id);
10238         if (!l)
10239                 return -EINVAL;
10240
10241         strcpy(learner->name, l->name);
10242
10243         learner->n_match_fields = l->n_fields;
10244         learner->n_actions = l->n_actions;
10245         learner->default_action_is_const = l->default_action_is_const;
10246         learner->size = l->size;
10247         learner->n_key_timeouts = l->n_timeouts;
10248
10249         return 0;
10250 }
10251
10252 int
10253 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10254                                          uint32_t learner_id,
10255                                          uint32_t match_field_id,
10256                                          struct rte_swx_ctl_table_match_field_info *match_field)
10257 {
10258         struct learner *l;
10259         struct field *f;
10260
10261         if (!p || (learner_id >= p->n_learners) || !match_field)
10262                 return -EINVAL;
10263
10264         l = learner_find_by_id(p, learner_id);
10265         if (!l || (match_field_id >= l->n_fields))
10266                 return -EINVAL;
10267
10268         f = l->fields[match_field_id];
10269         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10270         match_field->is_header = l->header ? 1 : 0;
10271         match_field->n_bits = f->n_bits;
10272         match_field->offset = f->offset;
10273
10274         return 0;
10275 }
10276
10277 int
10278 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10279                                     uint32_t learner_id,
10280                                     uint32_t learner_action_id,
10281                                     struct rte_swx_ctl_table_action_info *learner_action)
10282 {
10283         struct learner *l;
10284
10285         if (!p || (learner_id >= p->n_learners) || !learner_action)
10286                 return -EINVAL;
10287
10288         l = learner_find_by_id(p, learner_id);
10289         if (!l || (learner_action_id >= l->n_actions))
10290                 return -EINVAL;
10291
10292         learner_action->action_id = l->actions[learner_action_id]->id;
10293
10294         learner_action->action_is_for_table_entries =
10295                 l->action_is_for_table_entries[learner_action_id];
10296
10297         learner_action->action_is_for_default_entry =
10298                 l->action_is_for_default_entry[learner_action_id];
10299
10300         return 0;
10301 }
10302
10303 int
10304 rte_swx_ctl_pipeline_learner_timeout_get(struct rte_swx_pipeline *p,
10305                                          uint32_t learner_id,
10306                                          uint32_t timeout_id,
10307                                          uint32_t *timeout)
10308 {
10309         struct learner *l;
10310
10311         if (!p || (learner_id >= p->n_learners) || !timeout)
10312                 return -EINVAL;
10313
10314         l = learner_find_by_id(p, learner_id);
10315         if (!l || (timeout_id >= l->n_timeouts))
10316                 return -EINVAL;
10317
10318         *timeout = l->timeout[timeout_id];
10319         return 0;
10320 }
10321
10322 int
10323 rte_swx_ctl_pipeline_learner_timeout_set(struct rte_swx_pipeline *p,
10324                                          uint32_t learner_id,
10325                                          uint32_t timeout_id,
10326                                          uint32_t timeout)
10327 {
10328         struct learner *l;
10329         struct rte_swx_table_state *ts;
10330         int status;
10331
10332         if (!p || (learner_id >= p->n_learners) || !timeout)
10333                 return -EINVAL;
10334
10335         l = learner_find_by_id(p, learner_id);
10336         if (!l || (timeout_id >= l->n_timeouts))
10337                 return -EINVAL;
10338
10339         if (!p->build_done)
10340                 return -EINVAL;
10341
10342         ts = &p->table_state[p->n_tables + p->n_selectors + l->id];
10343
10344         status = rte_swx_table_learner_timeout_update(ts->obj, timeout_id, timeout);
10345         if (status)
10346                 return -EINVAL;
10347
10348         l->timeout[timeout_id] = timeout;
10349
10350         return 0;
10351 }
10352
10353 int
10354 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10355                                  struct rte_swx_table_state **table_state)
10356 {
10357         if (!p || !table_state || !p->build_done)
10358                 return -EINVAL;
10359
10360         *table_state = p->table_state;
10361         return 0;
10362 }
10363
10364 int
10365 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10366                                  struct rte_swx_table_state *table_state)
10367 {
10368         if (!p || !table_state || !p->build_done)
10369                 return -EINVAL;
10370
10371         p->table_state = table_state;
10372         return 0;
10373 }
10374
10375 int
10376 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10377                                         uint32_t port_id,
10378                                         struct rte_swx_port_in_stats *stats)
10379 {
10380         struct port_in *port;
10381
10382         if (!p || !stats)
10383                 return -EINVAL;
10384
10385         port = port_in_find(p, port_id);
10386         if (!port)
10387                 return -EINVAL;
10388
10389         port->type->ops.stats_read(port->obj, stats);
10390         return 0;
10391 }
10392
10393 int
10394 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10395                                          uint32_t port_id,
10396                                          struct rte_swx_port_out_stats *stats)
10397 {
10398         struct port_out *port;
10399
10400         if (!p || !stats)
10401                 return -EINVAL;
10402
10403         port = port_out_find(p, port_id);
10404         if (!port)
10405                 return -EINVAL;
10406
10407         port->type->ops.stats_read(port->obj, stats);
10408         return 0;
10409 }
10410
10411 int
10412 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10413                                       const char *table_name,
10414                                       struct rte_swx_table_stats *stats)
10415 {
10416         struct table *table;
10417         struct table_statistics *table_stats;
10418
10419         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10420                 return -EINVAL;
10421
10422         table = table_find(p, table_name);
10423         if (!table)
10424                 return -EINVAL;
10425
10426         table_stats = &p->table_stats[table->id];
10427
10428         memcpy(stats->n_pkts_action,
10429                table_stats->n_pkts_action,
10430                p->n_actions * sizeof(uint64_t));
10431
10432         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10433         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10434
10435         return 0;
10436 }
10437
10438 int
10439 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10440         const char *selector_name,
10441         struct rte_swx_pipeline_selector_stats *stats)
10442 {
10443         struct selector *s;
10444
10445         if (!p || !selector_name || !selector_name[0] || !stats)
10446                 return -EINVAL;
10447
10448         s = selector_find(p, selector_name);
10449         if (!s)
10450                 return -EINVAL;
10451
10452         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10453
10454         return 0;
10455 }
10456
10457 int
10458 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10459                                         const char *learner_name,
10460                                         struct rte_swx_learner_stats *stats)
10461 {
10462         struct learner *l;
10463         struct learner_statistics *learner_stats;
10464
10465         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10466                 return -EINVAL;
10467
10468         l = learner_find(p, learner_name);
10469         if (!l)
10470                 return -EINVAL;
10471
10472         learner_stats = &p->learner_stats[l->id];
10473
10474         memcpy(stats->n_pkts_action,
10475                learner_stats->n_pkts_action,
10476                p->n_actions * sizeof(uint64_t));
10477
10478         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10479         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10480
10481         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10482         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10483
10484         stats->n_pkts_rearm = learner_stats->n_pkts_rearm;
10485         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10486
10487         return 0;
10488 }
10489
10490 int
10491 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10492                               uint32_t regarray_id,
10493                               struct rte_swx_ctl_regarray_info *regarray)
10494 {
10495         struct regarray *r;
10496
10497         if (!p || !regarray)
10498                 return -EINVAL;
10499
10500         r = regarray_find_by_id(p, regarray_id);
10501         if (!r)
10502                 return -EINVAL;
10503
10504         strcpy(regarray->name, r->name);
10505         regarray->size = r->size;
10506         return 0;
10507 }
10508
10509 int
10510 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10511                                    const char *regarray_name,
10512                                    uint32_t regarray_index,
10513                                    uint64_t *value)
10514 {
10515         struct regarray *regarray;
10516         struct regarray_runtime *r;
10517
10518         if (!p || !regarray_name || !value)
10519                 return -EINVAL;
10520
10521         regarray = regarray_find(p, regarray_name);
10522         if (!regarray || (regarray_index >= regarray->size))
10523                 return -EINVAL;
10524
10525         r = &p->regarray_runtime[regarray->id];
10526         *value = r->regarray[regarray_index];
10527         return 0;
10528 }
10529
10530 int
10531 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10532                                    const char *regarray_name,
10533                                    uint32_t regarray_index,
10534                                    uint64_t value)
10535 {
10536         struct regarray *regarray;
10537         struct regarray_runtime *r;
10538
10539         if (!p || !regarray_name)
10540                 return -EINVAL;
10541
10542         regarray = regarray_find(p, regarray_name);
10543         if (!regarray || (regarray_index >= regarray->size))
10544                 return -EINVAL;
10545
10546         r = &p->regarray_runtime[regarray->id];
10547         r->regarray[regarray_index] = value;
10548         return 0;
10549 }
10550
10551 int
10552 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10553                               uint32_t metarray_id,
10554                               struct rte_swx_ctl_metarray_info *metarray)
10555 {
10556         struct metarray *m;
10557
10558         if (!p || !metarray)
10559                 return -EINVAL;
10560
10561         m = metarray_find_by_id(p, metarray_id);
10562         if (!m)
10563                 return -EINVAL;
10564
10565         strcpy(metarray->name, m->name);
10566         metarray->size = m->size;
10567         return 0;
10568 }
10569
10570 int
10571 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10572                               const char *name,
10573                               struct rte_meter_trtcm_params *params)
10574 {
10575         struct meter_profile *mp;
10576         int status;
10577
10578         CHECK(p, EINVAL);
10579         CHECK_NAME(name, EINVAL);
10580         CHECK(params, EINVAL);
10581         CHECK(!meter_profile_find(p, name), EEXIST);
10582
10583         /* Node allocation. */
10584         mp = calloc(1, sizeof(struct meter_profile));
10585         CHECK(mp, ENOMEM);
10586
10587         /* Node initialization. */
10588         strcpy(mp->name, name);
10589         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10590         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10591         if (status) {
10592                 free(mp);
10593                 CHECK(0, EINVAL);
10594         }
10595
10596         /* Node add to tailq. */
10597         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10598
10599         return 0;
10600 }
10601
10602 int
10603 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10604                                  const char *name)
10605 {
10606         struct meter_profile *mp;
10607
10608         CHECK(p, EINVAL);
10609         CHECK_NAME(name, EINVAL);
10610
10611         mp = meter_profile_find(p, name);
10612         CHECK(mp, EINVAL);
10613         CHECK(!mp->n_users, EBUSY);
10614
10615         /* Remove node from tailq. */
10616         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10617         free(mp);
10618
10619         return 0;
10620 }
10621
10622 int
10623 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10624                         const char *metarray_name,
10625                         uint32_t metarray_index)
10626 {
10627         struct meter_profile *mp_old;
10628         struct metarray *metarray;
10629         struct metarray_runtime *metarray_runtime;
10630         struct meter *m;
10631
10632         CHECK(p, EINVAL);
10633         CHECK_NAME(metarray_name, EINVAL);
10634
10635         metarray = metarray_find(p, metarray_name);
10636         CHECK(metarray, EINVAL);
10637         CHECK(metarray_index < metarray->size, EINVAL);
10638
10639         metarray_runtime = &p->metarray_runtime[metarray->id];
10640         m = &metarray_runtime->metarray[metarray_index];
10641         mp_old = m->profile;
10642
10643         meter_init(m);
10644
10645         mp_old->n_users--;
10646
10647         return 0;
10648 }
10649
10650 int
10651 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10652                       const char *metarray_name,
10653                       uint32_t metarray_index,
10654                       const char *profile_name)
10655 {
10656         struct meter_profile *mp, *mp_old;
10657         struct metarray *metarray;
10658         struct metarray_runtime *metarray_runtime;
10659         struct meter *m;
10660
10661         CHECK(p, EINVAL);
10662         CHECK_NAME(metarray_name, EINVAL);
10663
10664         metarray = metarray_find(p, metarray_name);
10665         CHECK(metarray, EINVAL);
10666         CHECK(metarray_index < metarray->size, EINVAL);
10667
10668         mp = meter_profile_find(p, profile_name);
10669         CHECK(mp, EINVAL);
10670
10671         metarray_runtime = &p->metarray_runtime[metarray->id];
10672         m = &metarray_runtime->metarray[metarray_index];
10673         mp_old = m->profile;
10674
10675         memset(m, 0, sizeof(struct meter));
10676         rte_meter_trtcm_config(&m->m, &mp->profile);
10677         m->profile = mp;
10678         m->color_mask = RTE_COLORS;
10679
10680         mp->n_users++;
10681         mp_old->n_users--;
10682
10683         return 0;
10684 }
10685
10686 int
10687 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10688                              const char *metarray_name,
10689                              uint32_t metarray_index,
10690                              struct rte_swx_ctl_meter_stats *stats)
10691 {
10692         struct metarray *metarray;
10693         struct metarray_runtime *metarray_runtime;
10694         struct meter *m;
10695
10696         CHECK(p, EINVAL);
10697         CHECK_NAME(metarray_name, EINVAL);
10698
10699         metarray = metarray_find(p, metarray_name);
10700         CHECK(metarray, EINVAL);
10701         CHECK(metarray_index < metarray->size, EINVAL);
10702
10703         CHECK(stats, EINVAL);
10704
10705         metarray_runtime = &p->metarray_runtime[metarray->id];
10706         m = &metarray_runtime->metarray[metarray_index];
10707
10708         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10709         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10710
10711         return 0;
10712 }
10713
10714 int
10715 rte_swx_ctl_pipeline_mirroring_session_set(struct rte_swx_pipeline *p,
10716                                            uint32_t session_id,
10717                                            struct rte_swx_pipeline_mirroring_session_params *params)
10718 {
10719         struct mirroring_session *s;
10720
10721         CHECK(p, EINVAL);
10722         CHECK(p->build_done, EEXIST);
10723         CHECK(session_id < p->n_mirroring_sessions, EINVAL);
10724         CHECK(params, EINVAL);
10725         CHECK(params->port_id < p->n_ports_out, EINVAL);
10726
10727         s = &p->mirroring_sessions[session_id];
10728         s->port_id = params->port_id;
10729         s->fast_clone = params->fast_clone;
10730         s->truncation_length = params->truncation_length ? params->truncation_length : UINT32_MAX;
10731
10732         return 0;
10733 }
10734
10735 /*
10736  * Pipeline compilation.
10737  */
10738 static const char *
10739 instr_type_to_name(struct instruction *instr)
10740 {
10741         switch (instr->type) {
10742         case INSTR_RX: return "INSTR_RX";
10743
10744         case INSTR_TX: return "INSTR_TX";
10745         case INSTR_TX_I: return "INSTR_TX_I";
10746         case INSTR_DROP: return "INSTR_DROP";
10747         case INSTR_MIRROR: return "INSTR_MIRROR";
10748         case INSTR_RECIRCULATE: return "INSTR_RECIRCULATE";
10749         case INSTR_RECIRCID: return "INSTR_RECIRCID";
10750
10751         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10752         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10753         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10754         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10755         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10756         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10757         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10758         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10759
10760         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10761
10762         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10763
10764         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10765         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10766         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10767         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10768         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10769         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10770         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10771         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10772         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10773
10774         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10775         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10776
10777         case INSTR_MOV: return "INSTR_MOV";
10778         case INSTR_MOV_MH: return "INSTR_MOV_MH";
10779         case INSTR_MOV_HM: return "INSTR_MOV_HM";
10780         case INSTR_MOV_HH: return "INSTR_MOV_HH";
10781         case INSTR_MOV_I: return "INSTR_MOV_I";
10782
10783         case INSTR_DMA_HT: return "INSTR_DMA_HT";
10784         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10785         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10786         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10787         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10788         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10789         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10790         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10791
10792         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10793         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10794         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10795         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10796         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10797         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10798
10799         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10800         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10801         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10802         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10803         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10804         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10805
10806         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10807         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10808         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10809         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10810
10811         case INSTR_ALU_AND: return "INSTR_ALU_AND";
10812         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10813         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10814         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10815         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10816
10817         case INSTR_ALU_OR: return "INSTR_ALU_OR";
10818         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10819         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10820         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10821         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10822
10823         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10824         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10825         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10826         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10827         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10828
10829         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10830         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10831         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10832         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10833         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10834         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10835
10836         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10837         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10838         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10839         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10840         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10841         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10842
10843         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10844         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10845         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10846
10847         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10848         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10849         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10850         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10851         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10852         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10853
10854         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10855         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10856         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10857         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10858         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10859         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10860         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10861         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10862         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10863
10864         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10865         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10866         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10867         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10868         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10869         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10870         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10871         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10872         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10873
10874         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10875         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10876         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10877
10878         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10879         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10880         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10881         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10882         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10883         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10884         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10885         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10886         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10887         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10888         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10889         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10890
10891         case INSTR_TABLE: return "INSTR_TABLE";
10892         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10893         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10894         case INSTR_LEARNER: return "INSTR_LEARNER";
10895         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10896
10897         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10898         case INSTR_LEARNER_REARM: return "INSTR_LEARNER_REARM";
10899         case INSTR_LEARNER_REARM_NEW: return "INSTR_LEARNER_REARM_NEW";
10900         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10901
10902         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10903         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10904         case INSTR_HASH_FUNC: return "INSTR_HASH_FUNC";
10905
10906         case INSTR_JMP: return "INSTR_JMP";
10907         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10908         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10909         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10910         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10911         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10912         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10913         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10914         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10915         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10916         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10917         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10918         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10919         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10920         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10921         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10922         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10923         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10924         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10925         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10926         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10927         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10928         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10929         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10930         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10931         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10932         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10933         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10934         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10935
10936         case INSTR_RETURN: return "INSTR_RETURN";
10937
10938         default: return "INSTR_UNKNOWN";
10939         }
10940 }
10941
10942 typedef void
10943 (*instruction_export_t)(struct instruction *, FILE *);
10944
10945 static void
10946 instr_io_export(struct instruction *instr, FILE *f)
10947 {
10948         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10949
10950         /* n_io, n_io_imm, n_hdrs. */
10951         if (instr->type == INSTR_RX ||
10952             instr->type == INSTR_TX ||
10953             instr->type == INSTR_HDR_EXTRACT_M ||
10954             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10955                 n_io = 1;
10956
10957         if (instr->type == INSTR_TX_I)
10958                 n_io_imm = 1;
10959
10960         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10961                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10962
10963         if (instr->type == INSTR_HDR_EXTRACT_M ||
10964             instr->type == INSTR_HDR_LOOKAHEAD ||
10965             instr->type == INSTR_HDR_EMIT)
10966                 n_hdrs = 1;
10967
10968         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10969                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10970
10971         /* instr. */
10972         fprintf(f,
10973                 "\t{\n"
10974                 "\t\t.type = %s,\n",
10975                 instr_type_to_name(instr));
10976
10977         /* instr.io. */
10978         if (n_io || n_io_imm || n_hdrs)
10979                 fprintf(f,
10980                         "\t\t.io = {\n");
10981
10982         /* instr.io.io. */
10983         if (n_io)
10984                 fprintf(f,
10985                         "\t\t\t.io = {\n"
10986                         "\t\t\t\t.offset = %u,\n"
10987                         "\t\t\t\t.n_bits = %u,\n"
10988                         "\t\t\t},\n",
10989                         instr->io.io.offset,
10990                         instr->io.io.n_bits);
10991
10992         if (n_io_imm)
10993                 fprintf(f,
10994                         "\t\t\t.io = {\n"
10995                         "\t\t\t\t.val = %u,\n"
10996                         "\t\t\t},\n",
10997                         instr->io.io.val);
10998
10999         /* instr.io.hdr. */
11000         if (n_hdrs) {
11001                 fprintf(f,
11002                         "\t\t.hdr = {\n");
11003
11004                 /* instr.io.hdr.header_id. */
11005                 fprintf(f,
11006                         "\t\t\t.header_id = {");
11007
11008                 for (i = 0; i < n_hdrs; i++)
11009                         fprintf(f,
11010                                 "%u, ",
11011                                 instr->io.hdr.header_id[i]);
11012
11013                 fprintf(f,
11014                         "},\n");
11015
11016                 /* instr.io.hdr.struct_id. */
11017                 fprintf(f,
11018                         "\t\t\t.struct_id = {");
11019
11020                 for (i = 0; i < n_hdrs; i++)
11021                         fprintf(f,
11022                                 "%u, ",
11023                                 instr->io.hdr.struct_id[i]);
11024
11025                 fprintf(f,
11026                         "},\n");
11027
11028                 /* instr.io.hdr.n_bytes. */
11029                 fprintf(f,
11030                         "\t\t\t.n_bytes = {");
11031
11032                 for (i = 0; i < n_hdrs; i++)
11033                         fprintf(f,
11034                                 "%u, ",
11035                                 instr->io.hdr.n_bytes[i]);
11036
11037                 fprintf(f,
11038                         "},\n");
11039
11040                 /* instr.io.hdr - closing curly brace. */
11041                 fprintf(f,
11042                         "\t\t\t}\n,");
11043         }
11044
11045         /* instr.io - closing curly brace. */
11046         if (n_io || n_io_imm || n_hdrs)
11047                 fprintf(f,
11048                         "\t\t},\n");
11049
11050         /* instr - closing curly brace. */
11051         fprintf(f,
11052                 "\t},\n");
11053 }
11054
11055 static void
11056 instr_mirror_export(struct instruction *instr, FILE *f)
11057 {
11058         fprintf(f,
11059                 "\t{\n"
11060                 "\t\t.type = %s,\n"
11061                 "\t\t.mirror = {\n"
11062                 "\t\t\t.dst = {\n"
11063                 "\t\t\t\t.struct_id = %u,\n"
11064                 "\t\t\t\t.n_bits = %u,\n"
11065                 "\t\t\t\t.offset = %u,\n"
11066                 "\t\t\t}\n,"
11067                 "\t\t\t.src = {\n"
11068                 "\t\t\t\t.struct_id = %u,\n"
11069                 "\t\t\t\t.n_bits = %u,\n"
11070                 "\t\t\t\t.offset = %u,\n"
11071                 "\t\t\t}\n,"
11072                 "\t\t},\n"
11073                 "\t},\n",
11074                 instr_type_to_name(instr),
11075                 instr->mirror.dst.struct_id,
11076                 instr->mirror.dst.n_bits,
11077                 instr->mirror.dst.offset,
11078                 instr->mirror.src.struct_id,
11079                 instr->mirror.src.n_bits,
11080                 instr->mirror.src.offset);
11081 }
11082
11083 static void
11084 instr_recirculate_export(struct instruction *instr, FILE *f)
11085 {
11086         fprintf(f,
11087                 "\t{\n"
11088                 "\t\t.type = %s,\n"
11089                 "\t},\n",
11090                 instr_type_to_name(instr));
11091 }
11092
11093 static void
11094 instr_recircid_export(struct instruction *instr, FILE *f)
11095 {
11096         fprintf(f,
11097                 "\t{\n"
11098                 "\t\t.type = %s,\n"
11099                 "\t\t.io = {\n"
11100                 "\t\t\t.offset = %u,\n"
11101                 "\t\t\t.n_bits = %u,\n"
11102                 "\t\t},\n"
11103                 "\t},\n",
11104                 instr_type_to_name(instr),
11105                 instr->io.io.offset,
11106                 instr->io.io.n_bits);
11107 }
11108
11109 static void
11110 instr_hdr_validate_export(struct instruction *instr, FILE *f)
11111 {
11112         fprintf(f,
11113                 "\t{\n"
11114                 "\t\t.type = %s,\n"
11115                 "\t\t.valid = {\n"
11116                 "\t\t\t.header_id = %u,\n"
11117                 "\t\t},\n"
11118                 "\t},\n",
11119                 instr_type_to_name(instr),
11120                 instr->valid.header_id);
11121 }
11122
11123 static void
11124 instr_mov_export(struct instruction *instr, FILE *f)
11125 {
11126         if (instr->type != INSTR_MOV_I)
11127                 fprintf(f,
11128                         "\t{\n"
11129                         "\t\t.type = %s,\n"
11130                         "\t\t.mov = {\n"
11131                         "\t\t\t.dst = {\n"
11132                         "\t\t\t\t.struct_id = %u,\n"
11133                         "\t\t\t\t.n_bits = %u,\n"
11134                         "\t\t\t\t.offset = %u,\n"
11135                         "\t\t\t},\n"
11136                         "\t\t\t.src = {\n"
11137                         "\t\t\t\t.struct_id = %u,\n"
11138                         "\t\t\t\t.n_bits = %u,\n"
11139                         "\t\t\t\t.offset = %u,\n"
11140                         "\t\t\t},\n"
11141                         "\t\t},\n"
11142                         "\t},\n",
11143                         instr_type_to_name(instr),
11144                         instr->mov.dst.struct_id,
11145                         instr->mov.dst.n_bits,
11146                         instr->mov.dst.offset,
11147                         instr->mov.src.struct_id,
11148                         instr->mov.src.n_bits,
11149                         instr->mov.src.offset);
11150         else
11151                 fprintf(f,
11152                         "\t{\n"
11153                         "\t\t.type = %s,\n"
11154                         "\t\t.mov = {\n"
11155                         "\t\t\t.dst = {\n"
11156                         "\t\t\t\t.struct_id = %u,\n"
11157                         "\t\t\t\t.n_bits = %u,\n"
11158                         "\t\t\t\t.offset = %u,\n"
11159                         "\t\t\t}\n,"
11160                         "\t\t\t.src_val = %" PRIu64 ",\n"
11161                         "\t\t},\n"
11162                         "\t},\n",
11163                         instr_type_to_name(instr),
11164                         instr->mov.dst.struct_id,
11165                         instr->mov.dst.n_bits,
11166                         instr->mov.dst.offset,
11167                         instr->mov.src_val);
11168 }
11169
11170 static void
11171 instr_dma_ht_export(struct instruction *instr, FILE *f)
11172 {
11173         uint32_t n_dma = 0, i;
11174
11175         /* n_dma. */
11176         n_dma = 1 + (instr->type - INSTR_DMA_HT);
11177
11178         /* instr. */
11179         fprintf(f,
11180                 "\t{\n"
11181                 "\t\t.type = %s,\n",
11182                 instr_type_to_name(instr));
11183
11184         /* instr.dma. */
11185         fprintf(f,
11186                 "\t\t.dma = {\n");
11187
11188         /* instr.dma.dst. */
11189         fprintf(f,
11190                 "\t\t\t.dst = {\n");
11191
11192         /* instr.dma.dst.header_id. */
11193         fprintf(f,
11194                 "\t\t\t\t.header_id = {");
11195
11196         for (i = 0; i < n_dma; i++)
11197                 fprintf(f,
11198                         "%u, ",
11199                         instr->dma.dst.header_id[i]);
11200
11201         fprintf(f,
11202                 "},\n");
11203
11204         /* instr.dma.dst.struct_id. */
11205         fprintf(f,
11206                 "\t\t\t\t.struct_id = {");
11207
11208         for (i = 0; i < n_dma; i++)
11209                 fprintf(f,
11210                         "%u, ",
11211                         instr->dma.dst.struct_id[i]);
11212
11213         fprintf(f,
11214                 "},\n");
11215
11216         /* instr.dma.dst - closing curly brace. */
11217         fprintf(f,
11218                 "\t\t\t},\n");
11219
11220         /* instr.dma.src. */
11221         fprintf(f,
11222                 "\t\t\t.src = {\n");
11223
11224         /* instr.dma.src.offset. */
11225         fprintf(f,
11226                 "\t\t\t\t.offset = {");
11227
11228         for (i = 0; i < n_dma; i++)
11229                 fprintf(f,
11230                         "%u, ",
11231                         instr->dma.src.offset[i]);
11232
11233         fprintf(f,
11234                 "},\n");
11235
11236         /* instr.dma.src - closing curly brace. */
11237         fprintf(f,
11238                 "\t\t\t},\n");
11239
11240         /* instr.dma.n_bytes. */
11241         fprintf(f,
11242                 "\t\t\t.n_bytes = {");
11243
11244         for (i = 0; i < n_dma; i++)
11245                 fprintf(f,
11246                         "%u, ",
11247                         instr->dma.n_bytes[i]);
11248
11249         fprintf(f,
11250                 "},\n");
11251
11252         /* instr.dma - closing curly brace. */
11253         fprintf(f,
11254                 "\t\t},\n");
11255
11256         /* instr - closing curly brace. */
11257         fprintf(f,
11258                 "\t},\n");
11259 }
11260
11261 static void
11262 instr_alu_export(struct instruction *instr, FILE *f)
11263 {
11264         int imm = 0;
11265
11266         if (instr->type == INSTR_ALU_ADD_MI ||
11267             instr->type == INSTR_ALU_ADD_HI ||
11268             instr->type == INSTR_ALU_SUB_MI ||
11269             instr->type == INSTR_ALU_SUB_HI ||
11270             instr->type == INSTR_ALU_SHL_MI ||
11271             instr->type == INSTR_ALU_SHL_HI ||
11272             instr->type == INSTR_ALU_SHR_MI ||
11273             instr->type == INSTR_ALU_SHR_HI ||
11274             instr->type == INSTR_ALU_AND_I ||
11275             instr->type == INSTR_ALU_OR_I ||
11276             instr->type == INSTR_ALU_XOR_I)
11277                 imm = 1;
11278
11279         if (!imm)
11280                 fprintf(f,
11281                         "\t{\n"
11282                         "\t\t.type = %s,\n"
11283                         "\t\t.alu = {\n"
11284                         "\t\t\t.dst = {\n"
11285                         "\t\t\t\t.struct_id = %u,\n"
11286                         "\t\t\t\t.n_bits = %u,\n"
11287                         "\t\t\t\t.offset = %u,\n"
11288                         "\t\t\t},\n"
11289                         "\t\t\t.src = {\n"
11290                         "\t\t\t\t.struct_id = %u,\n"
11291                         "\t\t\t\t.n_bits = %u,\n"
11292                         "\t\t\t\t.offset = %u,\n"
11293                         "\t\t\t},\n"
11294                         "\t\t},\n"
11295                         "\t},\n",
11296                         instr_type_to_name(instr),
11297                         instr->alu.dst.struct_id,
11298                         instr->alu.dst.n_bits,
11299                         instr->alu.dst.offset,
11300                         instr->alu.src.struct_id,
11301                         instr->alu.src.n_bits,
11302                         instr->alu.src.offset);
11303         else
11304                 fprintf(f,
11305                         "\t{\n"
11306                         "\t\t.type = %s,\n"
11307                         "\t\t.alu = {\n"
11308                         "\t\t\t.dst = {\n"
11309                         "\t\t\t\t.struct_id = %u,\n"
11310                         "\t\t\t\t.n_bits = %u,\n"
11311                         "\t\t\t\t.offset = %u,\n"
11312                         "\t\t\t}\n,"
11313                         "\t\t\t.src_val = %" PRIu64 ",\n"
11314                         "\t\t},\n"
11315                         "\t},\n",
11316                         instr_type_to_name(instr),
11317                         instr->alu.dst.struct_id,
11318                         instr->alu.dst.n_bits,
11319                         instr->alu.dst.offset,
11320                         instr->alu.src_val);
11321 }
11322
11323 static void
11324 instr_hash_export(struct instruction *instr, FILE *f)
11325 {
11326         fprintf(f,
11327                 "\t{\n"
11328                 "\t\t.type = %s,\n"
11329                 "\t\t.hash_func = {\n"
11330                 "\t\t\t.hash_func_id = %u,\n"
11331                 "\t\t\t.dst = {\n"
11332                 "\t\t\t\t.offset = %u,\n"
11333                 "\t\t\t\t.n_bits = %u,\n"
11334                 "\t\t\t},\n"
11335                 "\t\t\t.src = {\n"
11336                 "\t\t\t\t.struct_id = %u,\n"
11337                 "\t\t\t\t.offset = %u,\n"
11338                 "\t\t\t\t.n_bytes = %u,\n"
11339                 "\t\t\t},\n"
11340                 "\t\t},\n"
11341                 "\t},\n",
11342                 instr_type_to_name(instr),
11343                 instr->hash_func.hash_func_id,
11344                 instr->hash_func.dst.offset,
11345                 instr->hash_func.dst.n_bits,
11346                 instr->hash_func.src.struct_id,
11347                 instr->hash_func.src.offset,
11348                 instr->hash_func.src.n_bytes);
11349 }
11350
11351 static void
11352 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11353 {
11354         int prefetch  = 0, idx_imm = 0, src_imm = 0;
11355
11356         if (instr->type == INSTR_REGPREFETCH_RH ||
11357             instr->type == INSTR_REGPREFETCH_RM ||
11358             instr->type == INSTR_REGPREFETCH_RI)
11359                 prefetch = 1;
11360
11361         /* index is the 3rd operand for the regrd instruction and the 2nd
11362          * operand for the regwr and regadd instructions.
11363          */
11364         if (instr->type == INSTR_REGPREFETCH_RI ||
11365             instr->type == INSTR_REGRD_HRI ||
11366             instr->type == INSTR_REGRD_MRI ||
11367             instr->type == INSTR_REGWR_RIH ||
11368             instr->type == INSTR_REGWR_RIM ||
11369             instr->type == INSTR_REGWR_RII ||
11370             instr->type == INSTR_REGADD_RIH ||
11371             instr->type == INSTR_REGADD_RIM ||
11372             instr->type == INSTR_REGADD_RII)
11373                 idx_imm = 1;
11374
11375         /* src is the 3rd operand for the regwr and regadd instructions. */
11376         if (instr->type == INSTR_REGWR_RHI ||
11377             instr->type == INSTR_REGWR_RMI ||
11378             instr->type == INSTR_REGWR_RII ||
11379             instr->type == INSTR_REGADD_RHI ||
11380             instr->type == INSTR_REGADD_RMI ||
11381             instr->type == INSTR_REGADD_RII)
11382                 src_imm = 1;
11383
11384         /* instr.regarray.regarray_id. */
11385         fprintf(f,
11386                 "\t{\n"
11387                 "\t\t.type = %s,\n"
11388                 "\t\t.regarray = {\n"
11389                 "\t\t\t.regarray_id = %u,\n",
11390                 instr_type_to_name(instr),
11391                 instr->regarray.regarray_id);
11392
11393         /* instr.regarray.idx / instr.regarray.idx_val. */
11394         if (!idx_imm)
11395                 fprintf(f,
11396                         "\t\t\t\t.idx = {\n"
11397                         "\t\t\t\t\t.struct_id = %u,\n"
11398                         "\t\t\t\t\t.n_bits = %u,\n"
11399                         "\t\t\t\t\t.offset = %u,\n"
11400                         "\t\t\t\t},\n",
11401                         instr->regarray.idx.struct_id,
11402                         instr->regarray.idx.n_bits,
11403                         instr->regarray.idx.offset);
11404         else
11405                 fprintf(f,
11406                         "\t\t\t\t.idx_val = %u,\n",
11407                         instr->regarray.idx_val);
11408
11409         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
11410         if (!prefetch) {
11411                 if (!src_imm)
11412                         fprintf(f,
11413                                 "\t\t\t\t.dstsrc = {\n"
11414                                 "\t\t\t\t\t.struct_id = %u,\n"
11415                                 "\t\t\t\t\t.n_bits = %u,\n"
11416                                 "\t\t\t\t\t.offset = %u,\n"
11417                                 "\t\t\t\t},\n",
11418                                 instr->regarray.dstsrc.struct_id,
11419                                 instr->regarray.dstsrc.n_bits,
11420                                 instr->regarray.dstsrc.offset);
11421                 else
11422                         fprintf(f,
11423                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
11424                                 instr->regarray.dstsrc_val);
11425         }
11426
11427         /* instr.regarray and instr - closing curly braces. */
11428         fprintf(f,
11429                 "\t\t},\n"
11430                 "\t},\n");
11431 }
11432
11433 static void
11434 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11435 {
11436         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
11437
11438         if (instr->type == INSTR_METPREFETCH_H ||
11439             instr->type == INSTR_METPREFETCH_M ||
11440             instr->type == INSTR_METPREFETCH_I)
11441                 prefetch = 1;
11442
11443         /* idx_imm. */
11444         if (instr->type == INSTR_METPREFETCH_I ||
11445             instr->type == INSTR_METER_IHM ||
11446             instr->type == INSTR_METER_IHI ||
11447             instr->type == INSTR_METER_IMM ||
11448             instr->type == INSTR_METER_IMI)
11449                 idx_imm = 1;
11450
11451         /* color_in_imm. */
11452         if (instr->type == INSTR_METER_HHI ||
11453             instr->type == INSTR_METER_HMI ||
11454             instr->type == INSTR_METER_MHI ||
11455             instr->type == INSTR_METER_MMI ||
11456             instr->type == INSTR_METER_IHI ||
11457             instr->type == INSTR_METER_IMI)
11458                 color_in_imm = 1;
11459
11460         /* instr.meter.metarray_id. */
11461         fprintf(f,
11462                 "\t{\n"
11463                 "\t\t.type = %s,\n"
11464                 "\t\t.meter = {\n"
11465                 "\t\t\t.metarray_id = %u,\n",
11466                 instr_type_to_name(instr),
11467                 instr->meter.metarray_id);
11468
11469         /* instr.meter.idx / instr.meter.idx_val. */
11470         if (!idx_imm)
11471                 fprintf(f,
11472                         "\t\t\t.idx = {\n"
11473                         "\t\t\t\t.struct_id = %u,\n"
11474                         "\t\t\t\t.n_bits = %u,\n"
11475                         "\t\t\t\t.offset = %u,\n"
11476                         "\t\t\t},\n",
11477                         instr->meter.idx.struct_id,
11478                         instr->meter.idx.n_bits,
11479                         instr->meter.idx.offset);
11480         else
11481                 fprintf(f,
11482                         "\t\t\t.idx_val = %u,\n",
11483                         instr->meter.idx_val);
11484
11485         if (!prefetch) {
11486                 /* instr.meter.length. */
11487                 fprintf(f,
11488                         "\t\t\t.length = {\n"
11489                         "\t\t\t\t.struct_id = %u,\n"
11490                         "\t\t\t\t.n_bits = %u,\n"
11491                         "\t\t\t\t.offset = %u,\n"
11492                         "\t\t\t},\n",
11493                         instr->meter.length.struct_id,
11494                         instr->meter.length.n_bits,
11495                         instr->meter.length.offset);
11496
11497                 /* instr.meter.color_in / instr.meter.color_in_val. */
11498                 if (!color_in_imm)
11499                         fprintf(f,
11500                                 "\t\t\t.color_in = {\n"
11501                                 "\t\t\t\t.struct_id = %u,\n"
11502                                 "\t\t\t\t.n_bits = %u,\n"
11503                                 "\t\t\t\t.offset = %u,\n"
11504                                 "\t\t\t},\n",
11505                                 instr->meter.color_in.struct_id,
11506                                 instr->meter.color_in.n_bits,
11507                                 instr->meter.color_in.offset);
11508                 else
11509                         fprintf(f,
11510                                 "\t\t\t.color_in_val = %u,\n",
11511                                 (uint32_t)instr->meter.color_in_val);
11512
11513                 /* instr.meter.color_out. */
11514                 fprintf(f,
11515                         "\t\t\t.color_out = {\n"
11516                         "\t\t\t\t.struct_id = %u,\n"
11517                         "\t\t\t\t.n_bits = %u,\n"
11518                         "\t\t\t\t.offset = %u,\n"
11519                         "\t\t\t},\n",
11520                         instr->meter.color_out.struct_id,
11521                         instr->meter.color_out.n_bits,
11522                         instr->meter.color_out.offset);
11523         }
11524
11525         /* instr.meter and instr - closing curly braces. */
11526         fprintf(f,
11527                 "\t\t},\n"
11528                 "\t},\n");
11529 }
11530
11531 static void
11532 instr_table_export(struct instruction *instr,
11533                 FILE *f)
11534 {
11535         fprintf(f,
11536                 "\t{\n"
11537                 "\t\t.type = %s,\n"
11538                 "\t\t.table = {\n"
11539                 "\t\t\t.table_id = %u,\n"
11540                 "\t\t},\n"
11541                 "\t},\n",
11542                 instr_type_to_name(instr),
11543                 instr->table.table_id);
11544 }
11545
11546 static void
11547 instr_learn_export(struct instruction *instr, FILE *f)
11548 {
11549         fprintf(f,
11550                 "\t{\n"
11551                 "\t\t.type = %s,\n"
11552                 "\t\t.learn = {\n"
11553                 "\t\t\t.action_id = %u,\n"
11554                 "\t\t\t.mf_first_arg_offset = %u,\n"
11555                 "\t\t\t.mf_timeout_id_offset = %u,\n"
11556                 "\t\t\t.mf_timeout_id_n_bits = %u,\n"
11557                 "\t\t},\n"
11558                 "\t},\n",
11559                 instr_type_to_name(instr),
11560                 instr->learn.action_id,
11561                 instr->learn.mf_first_arg_offset,
11562                 instr->learn.mf_timeout_id_offset,
11563                 instr->learn.mf_timeout_id_n_bits);
11564 }
11565
11566 static void
11567 instr_rearm_export(struct instruction *instr, FILE *f)
11568 {
11569         if (instr->type == INSTR_LEARNER_REARM)
11570                 fprintf(f,
11571                         "\t{\n"
11572                         "\t\t.type = %s,\n"
11573                         "\t},\n",
11574                         instr_type_to_name(instr));
11575         else
11576                 fprintf(f,
11577                         "\t{\n"
11578                         "\t\t.type = %s,\n"
11579                         "\t\t.learn = {\n"
11580                         "\t\t\t.mf_timeout_id_offset = %u,\n"
11581                         "\t\t\t.mf_timeout_id_n_bits = %u,\n"
11582                         "\t\t},\n"
11583                         "\t},\n",
11584                         instr_type_to_name(instr),
11585                         instr->learn.mf_timeout_id_offset,
11586                         instr->learn.mf_timeout_id_n_bits);
11587 }
11588
11589 static void
11590 instr_forget_export(struct instruction *instr, FILE *f)
11591 {
11592         fprintf(f,
11593                 "\t{\n"
11594                 "\t\t.type = %s,\n"
11595                 "\t},\n",
11596                 instr_type_to_name(instr));
11597 }
11598
11599 static void
11600 instr_extern_export(struct instruction *instr, FILE *f)
11601 {
11602         if (instr->type == INSTR_EXTERN_OBJ)
11603                 fprintf(f,
11604                         "\t{\n"
11605                         "\t\t.type = %s,\n"
11606                         "\t\t.ext_obj = {\n"
11607                         "\t\t\t.ext_obj_id = %u,\n"
11608                         "\t\t\t.func_id = %u,\n"
11609                         "\t\t},\n"
11610                         "\t},\n",
11611                         instr_type_to_name(instr),
11612                         instr->ext_obj.ext_obj_id,
11613                         instr->ext_obj.func_id);
11614         else
11615                 fprintf(f,
11616                         "\t{\n"
11617                         "\t\t.type = %s,\n"
11618                         "\t\t.ext_func = {\n"
11619                         "\t\t\t.ext_func_id = %u,\n"
11620                         "\t\t},\n"
11621                         "\t},\n",
11622                         instr_type_to_name(instr),
11623                         instr->ext_func.ext_func_id);
11624 }
11625
11626 static void
11627 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
11628 {
11629         fprintf(f,
11630                 "\t{\n"
11631                 "\t\t.type = %s,\n"
11632                 "\t\t.jmp = {\n"
11633                 "\t\t\t.ip = NULL,\n",
11634                 instr_type_to_name(instr));
11635
11636         switch (instr->type) {
11637         case INSTR_JMP_VALID:
11638         case INSTR_JMP_INVALID:
11639                 fprintf(f,
11640                         "\t\t\t.header_id = %u,\n",
11641                         instr->jmp.header_id);
11642                 break;
11643
11644         case INSTR_JMP_ACTION_HIT:
11645         case INSTR_JMP_ACTION_MISS:
11646                 fprintf(f,
11647                         "\t\t\t.action_id = %u,\n",
11648                         instr->jmp.action_id);
11649                 break;
11650
11651         case INSTR_JMP_EQ:
11652         case INSTR_JMP_EQ_MH:
11653         case INSTR_JMP_EQ_HM:
11654         case INSTR_JMP_EQ_HH:
11655         case INSTR_JMP_NEQ:
11656         case INSTR_JMP_NEQ_MH:
11657         case INSTR_JMP_NEQ_HM:
11658         case INSTR_JMP_NEQ_HH:
11659         case INSTR_JMP_LT:
11660         case INSTR_JMP_LT_MH:
11661         case INSTR_JMP_LT_HM:
11662         case INSTR_JMP_LT_HH:
11663         case INSTR_JMP_GT:
11664         case INSTR_JMP_GT_MH:
11665         case INSTR_JMP_GT_HM:
11666         case INSTR_JMP_GT_HH:
11667                 fprintf(f,
11668                         "\t\t\t.a = {\n"
11669                         "\t\t\t\t.struct_id = %u,\n"
11670                         "\t\t\t\t.n_bits = %u,\n"
11671                         "\t\t\t\t.offset = %u,\n"
11672                         "\t\t\t},\n"
11673                         "\t\t\t.b = {\n"
11674                         "\t\t\t\t.struct_id = %u,\n"
11675                         "\t\t\t\t.n_bits = %u,\n"
11676                         "\t\t\t\t.offset = %u,\n"
11677                         "\t\t\t},\n",
11678                         instr->jmp.a.struct_id,
11679                         instr->jmp.a.n_bits,
11680                         instr->jmp.a.offset,
11681                         instr->jmp.b.struct_id,
11682                         instr->jmp.b.n_bits,
11683                         instr->jmp.b.offset);
11684                 break;
11685
11686         case INSTR_JMP_EQ_I:
11687         case INSTR_JMP_NEQ_I:
11688         case INSTR_JMP_LT_MI:
11689         case INSTR_JMP_LT_HI:
11690         case INSTR_JMP_GT_MI:
11691         case INSTR_JMP_GT_HI:
11692                 fprintf(f,
11693                         "\t\t\t.a = {\n"
11694                         "\t\t\t\t.struct_id = %u,\n"
11695                         "\t\t\t\t.n_bits = %u,\n"
11696                         "\t\t\t\t.offset = %u,\n"
11697                         "\t\t\t}\n,"
11698                         "\t\t\t.b_val = %" PRIu64 ",\n",
11699                         instr->jmp.a.struct_id,
11700                         instr->jmp.a.n_bits,
11701                         instr->jmp.a.offset,
11702                         instr->jmp.b_val);
11703                 break;
11704
11705         default:
11706                 break;
11707         }
11708
11709         fprintf(f,
11710                 "\t\t},\n"
11711                 "\t},\n");
11712 }
11713
11714 static void
11715 instr_return_export(struct instruction *instr,
11716                 FILE *f)
11717 {
11718         fprintf(f,
11719                 "\t{\n"
11720                 "\t\t.type = %s,\n",
11721                 instr_type_to_name(instr));
11722
11723         fprintf(f,
11724                 "\t},\n");
11725 }
11726
11727 static instruction_export_t export_table[] = {
11728         [INSTR_RX] = instr_io_export,
11729
11730         [INSTR_TX] = instr_io_export,
11731         [INSTR_TX_I] = instr_io_export,
11732         [INSTR_DROP] = instr_io_export,
11733         [INSTR_MIRROR] = instr_mirror_export,
11734         [INSTR_RECIRCULATE] = instr_recirculate_export,
11735         [INSTR_RECIRCID] = instr_recircid_export,
11736
11737         [INSTR_HDR_EXTRACT] = instr_io_export,
11738         [INSTR_HDR_EXTRACT2] = instr_io_export,
11739         [INSTR_HDR_EXTRACT3] = instr_io_export,
11740         [INSTR_HDR_EXTRACT4] = instr_io_export,
11741         [INSTR_HDR_EXTRACT5] = instr_io_export,
11742         [INSTR_HDR_EXTRACT6] = instr_io_export,
11743         [INSTR_HDR_EXTRACT7] = instr_io_export,
11744         [INSTR_HDR_EXTRACT8] = instr_io_export,
11745
11746         [INSTR_HDR_EXTRACT_M] = instr_io_export,
11747
11748         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
11749
11750         [INSTR_HDR_EMIT] = instr_io_export,
11751         [INSTR_HDR_EMIT_TX] = instr_io_export,
11752         [INSTR_HDR_EMIT2_TX] = instr_io_export,
11753         [INSTR_HDR_EMIT3_TX] = instr_io_export,
11754         [INSTR_HDR_EMIT4_TX] = instr_io_export,
11755         [INSTR_HDR_EMIT5_TX] = instr_io_export,
11756         [INSTR_HDR_EMIT6_TX] = instr_io_export,
11757         [INSTR_HDR_EMIT7_TX] = instr_io_export,
11758         [INSTR_HDR_EMIT8_TX] = instr_io_export,
11759
11760         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
11761         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
11762
11763         [INSTR_MOV] = instr_mov_export,
11764         [INSTR_MOV_MH] = instr_mov_export,
11765         [INSTR_MOV_HM] = instr_mov_export,
11766         [INSTR_MOV_HH] = instr_mov_export,
11767         [INSTR_MOV_I] = instr_mov_export,
11768
11769         [INSTR_DMA_HT]  = instr_dma_ht_export,
11770         [INSTR_DMA_HT2] = instr_dma_ht_export,
11771         [INSTR_DMA_HT3] = instr_dma_ht_export,
11772         [INSTR_DMA_HT4] = instr_dma_ht_export,
11773         [INSTR_DMA_HT5] = instr_dma_ht_export,
11774         [INSTR_DMA_HT6] = instr_dma_ht_export,
11775         [INSTR_DMA_HT7] = instr_dma_ht_export,
11776         [INSTR_DMA_HT8] = instr_dma_ht_export,
11777
11778         [INSTR_ALU_ADD] = instr_alu_export,
11779         [INSTR_ALU_ADD_MH] = instr_alu_export,
11780         [INSTR_ALU_ADD_HM] = instr_alu_export,
11781         [INSTR_ALU_ADD_HH] = instr_alu_export,
11782         [INSTR_ALU_ADD_MI] = instr_alu_export,
11783         [INSTR_ALU_ADD_HI] = instr_alu_export,
11784
11785         [INSTR_ALU_SUB] = instr_alu_export,
11786         [INSTR_ALU_SUB_MH] = instr_alu_export,
11787         [INSTR_ALU_SUB_HM] = instr_alu_export,
11788         [INSTR_ALU_SUB_HH] = instr_alu_export,
11789         [INSTR_ALU_SUB_MI] = instr_alu_export,
11790         [INSTR_ALU_SUB_HI] = instr_alu_export,
11791
11792         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
11793         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
11794         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
11795         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
11796
11797         [INSTR_ALU_AND] = instr_alu_export,
11798         [INSTR_ALU_AND_MH] = instr_alu_export,
11799         [INSTR_ALU_AND_HM] = instr_alu_export,
11800         [INSTR_ALU_AND_HH] = instr_alu_export,
11801         [INSTR_ALU_AND_I] = instr_alu_export,
11802
11803         [INSTR_ALU_OR] = instr_alu_export,
11804         [INSTR_ALU_OR_MH] = instr_alu_export,
11805         [INSTR_ALU_OR_HM] = instr_alu_export,
11806         [INSTR_ALU_OR_HH] = instr_alu_export,
11807         [INSTR_ALU_OR_I] = instr_alu_export,
11808
11809         [INSTR_ALU_XOR] = instr_alu_export,
11810         [INSTR_ALU_XOR_MH] = instr_alu_export,
11811         [INSTR_ALU_XOR_HM] = instr_alu_export,
11812         [INSTR_ALU_XOR_HH] = instr_alu_export,
11813         [INSTR_ALU_XOR_I] = instr_alu_export,
11814
11815         [INSTR_ALU_SHL] = instr_alu_export,
11816         [INSTR_ALU_SHL_MH] = instr_alu_export,
11817         [INSTR_ALU_SHL_HM] = instr_alu_export,
11818         [INSTR_ALU_SHL_HH] = instr_alu_export,
11819         [INSTR_ALU_SHL_MI] = instr_alu_export,
11820         [INSTR_ALU_SHL_HI] = instr_alu_export,
11821
11822         [INSTR_ALU_SHR] = instr_alu_export,
11823         [INSTR_ALU_SHR_MH] = instr_alu_export,
11824         [INSTR_ALU_SHR_HM] = instr_alu_export,
11825         [INSTR_ALU_SHR_HH] = instr_alu_export,
11826         [INSTR_ALU_SHR_MI] = instr_alu_export,
11827         [INSTR_ALU_SHR_HI] = instr_alu_export,
11828
11829         [INSTR_REGPREFETCH_RH] = instr_reg_export,
11830         [INSTR_REGPREFETCH_RM] = instr_reg_export,
11831         [INSTR_REGPREFETCH_RI] = instr_reg_export,
11832
11833         [INSTR_REGRD_HRH] = instr_reg_export,
11834         [INSTR_REGRD_HRM] = instr_reg_export,
11835         [INSTR_REGRD_MRH] = instr_reg_export,
11836         [INSTR_REGRD_MRM] = instr_reg_export,
11837         [INSTR_REGRD_HRI] = instr_reg_export,
11838         [INSTR_REGRD_MRI] = instr_reg_export,
11839
11840         [INSTR_REGWR_RHH] = instr_reg_export,
11841         [INSTR_REGWR_RHM] = instr_reg_export,
11842         [INSTR_REGWR_RMH] = instr_reg_export,
11843         [INSTR_REGWR_RMM] = instr_reg_export,
11844         [INSTR_REGWR_RHI] = instr_reg_export,
11845         [INSTR_REGWR_RMI] = instr_reg_export,
11846         [INSTR_REGWR_RIH] = instr_reg_export,
11847         [INSTR_REGWR_RIM] = instr_reg_export,
11848         [INSTR_REGWR_RII] = instr_reg_export,
11849
11850         [INSTR_REGADD_RHH] = instr_reg_export,
11851         [INSTR_REGADD_RHM] = instr_reg_export,
11852         [INSTR_REGADD_RMH] = instr_reg_export,
11853         [INSTR_REGADD_RMM] = instr_reg_export,
11854         [INSTR_REGADD_RHI] = instr_reg_export,
11855         [INSTR_REGADD_RMI] = instr_reg_export,
11856         [INSTR_REGADD_RIH] = instr_reg_export,
11857         [INSTR_REGADD_RIM] = instr_reg_export,
11858         [INSTR_REGADD_RII] = instr_reg_export,
11859
11860         [INSTR_METPREFETCH_H] = instr_meter_export,
11861         [INSTR_METPREFETCH_M] = instr_meter_export,
11862         [INSTR_METPREFETCH_I] = instr_meter_export,
11863
11864         [INSTR_METER_HHM] = instr_meter_export,
11865         [INSTR_METER_HHI] = instr_meter_export,
11866         [INSTR_METER_HMM] = instr_meter_export,
11867         [INSTR_METER_HMI] = instr_meter_export,
11868         [INSTR_METER_MHM] = instr_meter_export,
11869         [INSTR_METER_MHI] = instr_meter_export,
11870         [INSTR_METER_MMM] = instr_meter_export,
11871         [INSTR_METER_MMI] = instr_meter_export,
11872         [INSTR_METER_IHM] = instr_meter_export,
11873         [INSTR_METER_IHI] = instr_meter_export,
11874         [INSTR_METER_IMM] = instr_meter_export,
11875         [INSTR_METER_IMI] = instr_meter_export,
11876
11877         [INSTR_TABLE] = instr_table_export,
11878         [INSTR_TABLE_AF] = instr_table_export,
11879         [INSTR_SELECTOR] = instr_table_export,
11880         [INSTR_LEARNER] = instr_table_export,
11881         [INSTR_LEARNER_AF] = instr_table_export,
11882
11883         [INSTR_LEARNER_LEARN] = instr_learn_export,
11884         [INSTR_LEARNER_REARM] = instr_rearm_export,
11885         [INSTR_LEARNER_REARM_NEW] = instr_rearm_export,
11886         [INSTR_LEARNER_FORGET] = instr_forget_export,
11887
11888         [INSTR_EXTERN_OBJ] = instr_extern_export,
11889         [INSTR_EXTERN_FUNC] = instr_extern_export,
11890         [INSTR_HASH_FUNC] = instr_hash_export,
11891
11892         [INSTR_JMP] = instr_jmp_export,
11893         [INSTR_JMP_VALID] = instr_jmp_export,
11894         [INSTR_JMP_INVALID] = instr_jmp_export,
11895         [INSTR_JMP_HIT] = instr_jmp_export,
11896         [INSTR_JMP_MISS] = instr_jmp_export,
11897         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11898         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11899
11900         [INSTR_JMP_EQ] = instr_jmp_export,
11901         [INSTR_JMP_EQ_MH] = instr_jmp_export,
11902         [INSTR_JMP_EQ_HM] = instr_jmp_export,
11903         [INSTR_JMP_EQ_HH] = instr_jmp_export,
11904         [INSTR_JMP_EQ_I] = instr_jmp_export,
11905
11906         [INSTR_JMP_NEQ] = instr_jmp_export,
11907         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11908         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11909         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11910         [INSTR_JMP_NEQ_I] = instr_jmp_export,
11911
11912         [INSTR_JMP_LT] = instr_jmp_export,
11913         [INSTR_JMP_LT_MH] = instr_jmp_export,
11914         [INSTR_JMP_LT_HM] = instr_jmp_export,
11915         [INSTR_JMP_LT_HH] = instr_jmp_export,
11916         [INSTR_JMP_LT_MI] = instr_jmp_export,
11917         [INSTR_JMP_LT_HI] = instr_jmp_export,
11918
11919         [INSTR_JMP_GT] = instr_jmp_export,
11920         [INSTR_JMP_GT_MH] = instr_jmp_export,
11921         [INSTR_JMP_GT_HM] = instr_jmp_export,
11922         [INSTR_JMP_GT_HH] = instr_jmp_export,
11923         [INSTR_JMP_GT_MI] = instr_jmp_export,
11924         [INSTR_JMP_GT_HI] = instr_jmp_export,
11925
11926         [INSTR_RETURN] = instr_return_export,
11927 };
11928
11929 static void
11930 action_data_codegen(struct action *a, FILE *f)
11931 {
11932         uint32_t i;
11933
11934         fprintf(f,
11935                 "static const struct instruction action_%s_instructions[] = {\n",
11936                 a->name);
11937
11938         for (i = 0; i < a->n_instructions; i++) {
11939                 struct instruction *instr = &a->instructions[i];
11940                 instruction_export_t func = export_table[instr->type];
11941
11942                 func(instr, f);
11943         }
11944
11945         fprintf(f, "};\n");
11946 }
11947
11948 static const char *
11949 instr_type_to_func(struct instruction *instr)
11950 {
11951         switch (instr->type) {
11952         case INSTR_RX: return NULL;
11953
11954         case INSTR_TX: return "__instr_tx_exec";
11955         case INSTR_TX_I: return "__instr_tx_i_exec";
11956         case INSTR_DROP: return "__instr_drop_exec";
11957         case INSTR_MIRROR: return "__instr_mirror_exec";
11958         case INSTR_RECIRCULATE: return "__instr_recirculate_exec";
11959         case INSTR_RECIRCID: return "__instr_recircid_exec";
11960
11961         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11962         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11963         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11964         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11965         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11966         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11967         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11968         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11969
11970         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11971
11972         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11973
11974         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11975         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11976         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11977         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11978         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11979         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11980         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11981         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11982         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11983
11984         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11985         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11986
11987         case INSTR_MOV: return "__instr_mov_exec";
11988         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11989         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11990         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11991         case INSTR_MOV_I: return "__instr_mov_i_exec";
11992
11993         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11994         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11995         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11996         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11997         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11998         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11999         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
12000         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
12001
12002         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
12003         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
12004         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
12005         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
12006         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
12007         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
12008
12009         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
12010         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
12011         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
12012         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
12013         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
12014         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
12015
12016         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
12017         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
12018         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
12019         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
12020
12021         case INSTR_ALU_AND: return "__instr_alu_and_exec";
12022         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
12023         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
12024         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
12025         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
12026
12027         case INSTR_ALU_OR: return "__instr_alu_or_exec";
12028         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
12029         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
12030         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
12031         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
12032
12033         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
12034         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
12035         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
12036         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
12037         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
12038
12039         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
12040         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
12041         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
12042         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
12043         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
12044         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
12045
12046         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
12047         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
12048         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
12049         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
12050         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
12051         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
12052
12053         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
12054         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
12055         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
12056
12057         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
12058         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
12059         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
12060         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
12061         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
12062         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
12063
12064         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
12065         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
12066         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
12067         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
12068         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
12069         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
12070         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
12071         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
12072         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
12073
12074         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
12075         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
12076         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
12077         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
12078         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
12079         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
12080         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
12081         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
12082         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
12083
12084         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
12085         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
12086         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
12087
12088         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
12089         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
12090         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
12091         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
12092         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
12093         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
12094         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
12095         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
12096         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
12097         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
12098         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
12099         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
12100
12101         case INSTR_TABLE: return NULL;
12102         case INSTR_TABLE_AF: return NULL;
12103         case INSTR_SELECTOR: return NULL;
12104         case INSTR_LEARNER: return NULL;
12105         case INSTR_LEARNER_AF: return NULL;
12106
12107         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
12108         case INSTR_LEARNER_REARM: return "__instr_rearm_exec";
12109         case INSTR_LEARNER_REARM_NEW: return "__instr_rearm_new_exec";
12110         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
12111
12112         case INSTR_EXTERN_OBJ: return NULL;
12113         case INSTR_EXTERN_FUNC: return NULL;
12114         case INSTR_HASH_FUNC: return "__instr_hash_func_exec";
12115
12116         case INSTR_JMP: return NULL;
12117         case INSTR_JMP_VALID: return NULL;
12118         case INSTR_JMP_INVALID: return NULL;
12119         case INSTR_JMP_HIT: return NULL;
12120         case INSTR_JMP_MISS: return NULL;
12121         case INSTR_JMP_ACTION_HIT: return NULL;
12122         case INSTR_JMP_ACTION_MISS: return NULL;
12123         case INSTR_JMP_EQ: return NULL;
12124         case INSTR_JMP_EQ_MH: return NULL;
12125         case INSTR_JMP_EQ_HM: return NULL;
12126         case INSTR_JMP_EQ_HH: return NULL;
12127         case INSTR_JMP_EQ_I: return NULL;
12128         case INSTR_JMP_NEQ: return NULL;
12129         case INSTR_JMP_NEQ_MH: return NULL;
12130         case INSTR_JMP_NEQ_HM: return NULL;
12131         case INSTR_JMP_NEQ_HH: return NULL;
12132         case INSTR_JMP_NEQ_I: return NULL;
12133         case INSTR_JMP_LT: return NULL;
12134         case INSTR_JMP_LT_MH: return NULL;
12135         case INSTR_JMP_LT_HM: return NULL;
12136         case INSTR_JMP_LT_HH: return NULL;
12137         case INSTR_JMP_LT_MI: return NULL;
12138         case INSTR_JMP_LT_HI: return NULL;
12139         case INSTR_JMP_GT: return NULL;
12140         case INSTR_JMP_GT_MH: return NULL;
12141         case INSTR_JMP_GT_HM: return NULL;
12142         case INSTR_JMP_GT_HH: return NULL;
12143         case INSTR_JMP_GT_MI: return NULL;
12144         case INSTR_JMP_GT_HI: return NULL;
12145
12146         case INSTR_RETURN: return NULL;
12147
12148         default: return NULL;
12149         }
12150 }
12151
12152 static void
12153 action_instr_does_tx_codegen(struct action *a,
12154                         uint32_t instr_pos,
12155                         struct instruction *instr,
12156                         FILE *f)
12157 {
12158         fprintf(f,
12159                 "%s(p, t, &action_%s_instructions[%u]);\n"
12160                 "\tthread_ip_reset(p, t);\n"
12161                 "\tinstr_rx_exec(p);\n"
12162                 "\treturn;\n",
12163                 instr_type_to_func(instr),
12164                 a->name,
12165                 instr_pos);
12166 }
12167
12168 static void
12169 action_instr_extern_obj_codegen(struct action *a,
12170                                 uint32_t instr_pos,
12171                                 FILE *f)
12172 {
12173         fprintf(f,
12174                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
12175                 a->name,
12176                 instr_pos);
12177 }
12178
12179 static void
12180 action_instr_extern_func_codegen(struct action *a,
12181                                  uint32_t instr_pos,
12182                                  FILE *f)
12183 {
12184         fprintf(f,
12185                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
12186                 a->name,
12187                 instr_pos);
12188 }
12189
12190 static void
12191 action_instr_jmp_codegen(struct action *a,
12192                          uint32_t instr_pos,
12193                          struct instruction *instr,
12194                          struct instruction_data *data,
12195                          FILE *f)
12196 {
12197         switch (instr->type) {
12198         case INSTR_JMP:
12199                 fprintf(f,
12200                         "goto %s;\n",
12201                         data->jmp_label);
12202                 return;
12203
12204         case INSTR_JMP_VALID:
12205                 fprintf(f,
12206                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
12207                         "\t\tgoto %s;\n",
12208                         a->name,
12209                         instr_pos,
12210                         data->jmp_label);
12211                 return;
12212
12213         case INSTR_JMP_INVALID:
12214                 fprintf(f,
12215                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
12216                         "\t\tgoto %s;\n",
12217                         a->name,
12218                         instr_pos,
12219                         data->jmp_label);
12220                 return;
12221
12222         case INSTR_JMP_HIT:
12223                 fprintf(f,
12224                         "if (t->hit)\n"
12225                         "\t\tgoto %s;\n",
12226                         data->jmp_label);
12227                 return;
12228
12229         case INSTR_JMP_MISS:
12230                 fprintf(f,
12231                         "if (!t->hit)\n"
12232                         "\t\tgoto %s;\n",
12233                         data->jmp_label);
12234                 return;
12235
12236         case INSTR_JMP_ACTION_HIT:
12237                 fprintf(f,
12238                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
12239                         "\t\tgoto %s;\n",
12240                         a->name,
12241                         instr_pos,
12242                         data->jmp_label);
12243                 return;
12244
12245         case INSTR_JMP_ACTION_MISS:
12246                 fprintf(f,
12247                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
12248                         "\t\tgoto %s;\n",
12249                         a->name,
12250                         instr_pos,
12251                         data->jmp_label);
12252                 return;
12253
12254         case INSTR_JMP_EQ:
12255                 fprintf(f,
12256                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12257                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12258                         "\t\tgoto %s;\n",
12259                         a->name,
12260                         instr_pos,
12261                         a->name,
12262                         instr_pos,
12263                         data->jmp_label);
12264                 return;
12265
12266         case INSTR_JMP_EQ_MH:
12267                 fprintf(f,
12268                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12269                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12270                         "\t\tgoto %s;\n",
12271                         a->name,
12272                         instr_pos,
12273                         a->name,
12274                         instr_pos,
12275                         data->jmp_label);
12276                 return;
12277
12278         case INSTR_JMP_EQ_HM:
12279                 fprintf(f,
12280                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
12281                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12282                         "\t\tgoto %s;\n",
12283                         a->name,
12284                         instr_pos,
12285                         a->name,
12286                         instr_pos,
12287                         data->jmp_label);
12288                 return;
12289
12290         case INSTR_JMP_EQ_HH:
12291                 fprintf(f,
12292                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
12293                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12294                         "\t\tgoto %s;\n",
12295                         a->name,
12296                         instr_pos,
12297                         a->name,
12298                         instr_pos,
12299                         data->jmp_label);
12300                 return;
12301
12302         case INSTR_JMP_EQ_I:
12303                 fprintf(f,
12304                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12305                         "action_%s_instructions[%u].jmp.b_val)\n"
12306                         "\t\tgoto %s;\n",
12307                         a->name,
12308                         instr_pos,
12309                         a->name,
12310                         instr_pos,
12311                         data->jmp_label);
12312                 return;
12313
12314         case INSTR_JMP_NEQ:
12315                 fprintf(f,
12316                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12317                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12318                         "\t\tgoto %s;\n",
12319                         a->name,
12320                         instr_pos,
12321                         a->name,
12322                         instr_pos,
12323                         data->jmp_label);
12324                 return;
12325
12326         case INSTR_JMP_NEQ_MH:
12327                 fprintf(f,
12328                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12329                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12330                         "\t\tgoto %s;\n",
12331                         a->name,
12332                         instr_pos,
12333                         a->name,
12334                         instr_pos,
12335                         data->jmp_label);
12336                 return;
12337
12338         case INSTR_JMP_NEQ_HM:
12339                 fprintf(f,
12340                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
12341                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12342                         "\t\tgoto %s;\n",
12343                         a->name,
12344                         instr_pos,
12345                         a->name,
12346                         instr_pos,
12347                         data->jmp_label);
12348                 return;
12349
12350         case INSTR_JMP_NEQ_HH:
12351                 fprintf(f,
12352                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
12353                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12354                         "\t\tgoto %s;\n",
12355                         a->name,
12356                         instr_pos,
12357                         a->name,
12358                         instr_pos,
12359                         data->jmp_label);
12360                 return;
12361
12362         case INSTR_JMP_NEQ_I:
12363                 fprintf(f,
12364                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12365                         "action_%s_instructions[%u].jmp.b_val)\n"
12366                         "\t\tgoto %s;\n",
12367                         a->name,
12368                         instr_pos,
12369                         a->name,
12370                         instr_pos,
12371                         data->jmp_label);
12372                 return;
12373
12374         case INSTR_JMP_LT:
12375                 fprintf(f,
12376                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12377                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12378                         "\t\tgoto %s;\n",
12379                         a->name,
12380                         instr_pos,
12381                         a->name,
12382                         instr_pos,
12383                         data->jmp_label);
12384                 return;
12385
12386         case INSTR_JMP_LT_MH:
12387                 fprintf(f,
12388                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12389                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12390                         "\t\tgoto %s;\n",
12391                         a->name,
12392                         instr_pos,
12393                         a->name,
12394                         instr_pos,
12395                         data->jmp_label);
12396                 return;
12397
12398         case INSTR_JMP_LT_HM:
12399                 fprintf(f,
12400                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12401                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12402                         "\t\tgoto %s;\n",
12403                         a->name,
12404                         instr_pos,
12405                         a->name,
12406                         instr_pos,
12407                         data->jmp_label);
12408                 return;
12409
12410         case INSTR_JMP_LT_HH:
12411                 fprintf(f,
12412                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12413                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12414                         "\t\tgoto %s;\n",
12415                         a->name,
12416                         instr_pos,
12417                         a->name,
12418                         instr_pos,
12419                         data->jmp_label);
12420                 return;
12421
12422         case INSTR_JMP_LT_MI:
12423                 fprintf(f,
12424                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12425                         "action_%s_instructions[%u].jmp.b_val)\n"
12426                         "\t\tgoto %s;\n",
12427                         a->name,
12428                         instr_pos,
12429                         a->name,
12430                         instr_pos,
12431                         data->jmp_label);
12432                 return;
12433
12434         case INSTR_JMP_LT_HI:
12435                 fprintf(f,
12436                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12437                         "action_%s_instructions[%u].jmp.b_val)\n"
12438                         "\t\tgoto %s;\n",
12439                         a->name,
12440                         instr_pos,
12441                         a->name,
12442                         instr_pos,
12443                         data->jmp_label);
12444                 return;
12445
12446         case INSTR_JMP_GT:
12447                 fprintf(f,
12448                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12449                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12450                         "\t\tgoto %s;\n",
12451                         a->name,
12452                         instr_pos,
12453                         a->name,
12454                         instr_pos,
12455                         data->jmp_label);
12456                 return;
12457
12458         case INSTR_JMP_GT_MH:
12459                 fprintf(f,
12460                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12461                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12462                         "\t\tgoto %s;\n",
12463                         a->name,
12464                         instr_pos,
12465                         a->name,
12466                         instr_pos,
12467                         data->jmp_label);
12468                 return;
12469
12470         case INSTR_JMP_GT_HM:
12471                 fprintf(f,
12472                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12473                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12474                         "\t\tgoto %s;\n",
12475                         a->name,
12476                         instr_pos,
12477                         a->name,
12478                         instr_pos,
12479                         data->jmp_label);
12480                 return;
12481
12482         case INSTR_JMP_GT_HH:
12483                 fprintf(f,
12484                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12485                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12486                         "\t\tgoto %s;\n",
12487                         a->name,
12488                         instr_pos,
12489                         a->name,
12490                         instr_pos,
12491                         data->jmp_label);
12492                 return;
12493
12494         case INSTR_JMP_GT_MI:
12495                 fprintf(f,
12496                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12497                         "action_%s_instructions[%u].jmp.b_val)\n"
12498                         "\t\tgoto %s;\n",
12499                         a->name,
12500                         instr_pos,
12501                         a->name,
12502                         instr_pos,
12503                         data->jmp_label);
12504                 return;
12505
12506         case INSTR_JMP_GT_HI:
12507                 fprintf(f,
12508                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12509                         "action_%s_instructions[%u].jmp.b_val)\n"
12510                         "\t\tgoto %s;\n",
12511                         a->name,
12512                         instr_pos,
12513                         a->name,
12514                         instr_pos,
12515                         data->jmp_label);
12516                 return;
12517
12518         default:
12519                 return;
12520         }
12521 }
12522
12523 static void
12524 action_instr_return_codegen(FILE *f)
12525 {
12526         fprintf(f,
12527                 "return;\n");
12528 }
12529
12530 static void
12531 action_instr_codegen(struct action *a, FILE *f)
12532 {
12533         uint32_t i;
12534
12535         fprintf(f,
12536                 "void\n"
12537                 "action_%s_run(struct rte_swx_pipeline *p)\n"
12538                 "{\n"
12539                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12540                 "\n",
12541                 a->name);
12542
12543         for (i = 0; i < a->n_instructions; i++) {
12544                 struct instruction *instr = &a->instructions[i];
12545                 struct instruction_data *data = &a->instruction_data[i];
12546
12547                 /* Label, if present. */
12548                 if (data->label[0])
12549                         fprintf(f, "\n%s : ", data->label);
12550                 else
12551                         fprintf(f, "\n\t");
12552
12553                 /* TX instruction type. */
12554                 if (instruction_does_tx(instr)) {
12555                         action_instr_does_tx_codegen(a, i, instr, f);
12556                         continue;
12557                 }
12558
12559                 /* Extern object/function instruction type. */
12560                 if (instr->type == INSTR_EXTERN_OBJ) {
12561                         action_instr_extern_obj_codegen(a, i, f);
12562                         continue;
12563                 }
12564
12565                 if (instr->type == INSTR_EXTERN_FUNC) {
12566                         action_instr_extern_func_codegen(a, i, f);
12567                         continue;
12568                 }
12569
12570                 /* Jump instruction type. */
12571                 if (instruction_is_jmp(instr)) {
12572                         action_instr_jmp_codegen(a, i, instr, data, f);
12573                         continue;
12574                 }
12575
12576                 /* Return instruction type. */
12577                 if (instr->type == INSTR_RETURN) {
12578                         action_instr_return_codegen(f);
12579                         continue;
12580                 }
12581
12582                 /* Any other instruction type. */
12583                 fprintf(f,
12584                         "%s(p, t, &action_%s_instructions[%u]);\n",
12585                         instr_type_to_func(instr),
12586                         a->name,
12587                         i);
12588         }
12589
12590         fprintf(f, "}\n\n");
12591 }
12592
12593 struct instruction_group {
12594         TAILQ_ENTRY(instruction_group) node;
12595
12596         uint32_t group_id;
12597
12598         uint32_t first_instr_id;
12599
12600         uint32_t last_instr_id;
12601
12602         instr_exec_t func;
12603 };
12604
12605 TAILQ_HEAD(instruction_group_list, instruction_group);
12606
12607 static struct instruction_group *
12608 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
12609 {
12610         struct instruction_group *g;
12611
12612         TAILQ_FOREACH(g, igl, node)
12613                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
12614                         return g;
12615
12616         return NULL;
12617 }
12618
12619 static void
12620 instruction_group_list_free(struct instruction_group_list *igl)
12621 {
12622         if (!igl)
12623                 return;
12624
12625         for ( ; ; ) {
12626                 struct instruction_group *g;
12627
12628                 g = TAILQ_FIRST(igl);
12629                 if (!g)
12630                         break;
12631
12632                 TAILQ_REMOVE(igl, g, node);
12633                 free(g);
12634         }
12635
12636         free(igl);
12637 }
12638
12639 static struct instruction_group_list *
12640 instruction_group_list_create(struct rte_swx_pipeline *p)
12641 {
12642         struct instruction_group_list *igl = NULL;
12643         struct instruction_group *g = NULL;
12644         uint32_t n_groups = 0, i;
12645
12646         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
12647                 goto error;
12648
12649         /* List init. */
12650         igl = calloc(1, sizeof(struct instruction_group_list));
12651         if (!igl)
12652                 goto error;
12653
12654         TAILQ_INIT(igl);
12655
12656         /* Allocate the first group. */
12657         g = calloc(1, sizeof(struct instruction_group));
12658         if (!g)
12659                 goto error;
12660
12661         /* Iteration 1: Separate the instructions into groups based on the thread yield
12662          * instructions. Do not worry about the jump instructions at this point.
12663          */
12664         for (i = 0; i < p->n_instructions; i++) {
12665                 struct instruction *instr = &p->instructions[i];
12666
12667                 /* Check for thread yield instructions. */
12668                 if (!instruction_does_thread_yield(instr))
12669                         continue;
12670
12671                 /* If the current group contains at least one instruction, then finalize it (with
12672                  * the previous instruction), add it to the list and allocate a new group (that
12673                  * starts with the current instruction).
12674                  */
12675                 if (i - g->first_instr_id) {
12676                         /* Finalize the group. */
12677                         g->last_instr_id = i - 1;
12678
12679                         /* Add the group to the list. Advance the number of groups. */
12680                         TAILQ_INSERT_TAIL(igl, g, node);
12681                         n_groups++;
12682
12683                         /* Allocate a new group. */
12684                         g = calloc(1, sizeof(struct instruction_group));
12685                         if (!g)
12686                                 goto error;
12687
12688                         /* Initialize the new group. */
12689                         g->group_id = n_groups;
12690                         g->first_instr_id = i;
12691                 }
12692
12693                 /* Finalize the current group (with the current instruction, therefore this group
12694                  * contains just the current thread yield instruction), add it to the list and
12695                  * allocate a new group (that starts with the next instruction).
12696                  */
12697
12698                 /* Finalize the group. */
12699                 g->last_instr_id = i;
12700
12701                 /* Add the group to the list. Advance the number of groups. */
12702                 TAILQ_INSERT_TAIL(igl, g, node);
12703                 n_groups++;
12704
12705                 /* Allocate a new group. */
12706                 g = calloc(1, sizeof(struct instruction_group));
12707                 if (!g)
12708                         goto error;
12709
12710                 /* Initialize the new group. */
12711                 g->group_id = n_groups;
12712                 g->first_instr_id = i + 1;
12713         }
12714
12715         /* Handle the last group. */
12716         if (i - g->first_instr_id) {
12717                 /* Finalize the group. */
12718                 g->last_instr_id = i - 1;
12719
12720                 /* Add the group to the list. Advance the number of groups. */
12721                 TAILQ_INSERT_TAIL(igl, g, node);
12722                 n_groups++;
12723         } else
12724                 free(g);
12725
12726         g = NULL;
12727
12728         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
12729          * the destination of a jump instruction located in a different group ("far jump"), then the
12730          * current group has to be split, so that the instruction representing the far jump
12731          * destination is at the start of its group.
12732          */
12733         for ( ; ; ) {
12734                 int is_modified = 0;
12735
12736                 for (i = 0; i < p->n_instructions; i++) {
12737                         struct instruction_data *data = &p->instruction_data[i];
12738                         struct instruction_group *g;
12739                         uint32_t j;
12740
12741                         /* Continue when the current instruction is not a jump destination. */
12742                         if (!data->n_users)
12743                                 continue;
12744
12745                         g = instruction_group_list_group_find(igl, i);
12746                         if (!g)
12747                                 goto error;
12748
12749                         /* Find out all the jump instructions with this destination. */
12750                         for (j = 0; j < p->n_instructions; j++) {
12751                                 struct instruction *jmp_instr = &p->instructions[j];
12752                                 struct instruction_data *jmp_data = &p->instruction_data[j];
12753                                 struct instruction_group *jmp_g, *new_g;
12754
12755                                 /* Continue when not a jump instruction. Even when jump instruction,
12756                                  * continue when the jump destination is not this instruction.
12757                                  */
12758                                 if (!instruction_is_jmp(jmp_instr) ||
12759                                     strcmp(jmp_data->jmp_label, data->label))
12760                                         continue;
12761
12762                                 jmp_g = instruction_group_list_group_find(igl, j);
12763                                 if (!jmp_g)
12764                                         goto error;
12765
12766                                 /* Continue when both the jump instruction and the jump destination
12767                                  * instruction are in the same group. Even when in different groups,
12768                                  * still continue if the jump destination instruction is already the
12769                                  * first instruction of its group.
12770                                  */
12771                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
12772                                         continue;
12773
12774                                 /* Split the group of the current jump destination instruction to
12775                                  * make this instruction the first instruction of a new group.
12776                                  */
12777                                 new_g = calloc(1, sizeof(struct instruction_group));
12778                                 if (!new_g)
12779                                         goto error;
12780
12781                                 new_g->group_id = n_groups;
12782                                 new_g->first_instr_id = i;
12783                                 new_g->last_instr_id = g->last_instr_id;
12784
12785                                 g->last_instr_id = i - 1;
12786
12787                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
12788                                 n_groups++;
12789                                 is_modified = 1;
12790
12791                                 /* The decision to split this group (to make the current instruction
12792                                  * the first instruction of a new group) is already taken and fully
12793                                  * implemented, so no need to search for more reasons to do it.
12794                                  */
12795                                 break;
12796                         }
12797                 }
12798
12799                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
12800                  * previously considered local (i.e. the jump destination is in the same group as
12801                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
12802                  * different group than the jump instruction). Wost case scenario: each instruction
12803                  * that is a jump destination ends up as the first instruction of its group.
12804                  */
12805                 if (!is_modified)
12806                         break;
12807         }
12808
12809         /* Re-assign the group IDs to be in incremental order. */
12810         i = 0;
12811         TAILQ_FOREACH(g, igl, node) {
12812                 g->group_id = i;
12813
12814                 i++;
12815         }
12816
12817         return igl;
12818
12819 error:
12820         instruction_group_list_free(igl);
12821
12822         free(g);
12823
12824         return NULL;
12825 }
12826
12827 static void
12828 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12829                                uint32_t instr_pos,
12830                                struct instruction *instr,
12831                                FILE *f)
12832 {
12833         fprintf(f,
12834                 "%s(p, t, &pipeline_instructions[%u]);\n"
12835                 "\tthread_ip_reset(p, t);\n"
12836                 "\tinstr_rx_exec(p);\n"
12837                 "\treturn;\n",
12838                 instr_type_to_func(instr),
12839                 instr_pos);
12840 }
12841
12842 static int
12843 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12844                            struct instruction_group_list *igl,
12845                            uint32_t jmp_instr_id,
12846                            struct instruction *jmp_instr,
12847                            struct instruction_data *jmp_data,
12848                            FILE *f)
12849 {
12850         struct instruction_group *jmp_g, *g;
12851         struct instruction_data *data;
12852         uint32_t instr_id;
12853
12854         switch (jmp_instr->type) {
12855         case INSTR_JMP:
12856                 break;
12857
12858         case INSTR_JMP_VALID:
12859                 fprintf(f,
12860                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12861                         jmp_instr_id);
12862                 break;
12863
12864         case INSTR_JMP_INVALID:
12865                 fprintf(f,
12866                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12867                         jmp_instr_id);
12868                 break;
12869
12870         case INSTR_JMP_HIT:
12871                 fprintf(f,
12872                         "if (t->hit)\n");
12873                 break;
12874
12875         case INSTR_JMP_MISS:
12876                 fprintf(f,
12877                         "if (!t->hit)\n");
12878                 break;
12879
12880         case INSTR_JMP_ACTION_HIT:
12881                 fprintf(f,
12882                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12883                         jmp_instr_id);
12884                 break;
12885
12886         case INSTR_JMP_ACTION_MISS:
12887                 fprintf(f,
12888                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12889                         jmp_instr_id);
12890                 break;
12891
12892         case INSTR_JMP_EQ:
12893                 fprintf(f,
12894                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12895                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12896                         jmp_instr_id,
12897                         jmp_instr_id);
12898                 break;
12899
12900         case INSTR_JMP_EQ_MH:
12901                 fprintf(f,
12902                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12903                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12904                         jmp_instr_id,
12905                         jmp_instr_id);
12906                 break;
12907
12908         case INSTR_JMP_EQ_HM:
12909                 fprintf(f,
12910                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12911                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12912                         jmp_instr_id,
12913                         jmp_instr_id);
12914                 break;
12915
12916         case INSTR_JMP_EQ_HH:
12917                 fprintf(f,
12918                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12919                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12920                         jmp_instr_id,
12921                         jmp_instr_id);
12922                 break;
12923
12924         case INSTR_JMP_EQ_I:
12925                 fprintf(f,
12926                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12927                         "pipeline_instructions[%u].jmp.b_val)",
12928                         jmp_instr_id,
12929                         jmp_instr_id);
12930                 break;
12931
12932         case INSTR_JMP_NEQ:
12933                 fprintf(f,
12934                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12935                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12936                         jmp_instr_id,
12937                         jmp_instr_id);
12938                 break;
12939
12940         case INSTR_JMP_NEQ_MH:
12941                 fprintf(f,
12942                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12943                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12944                         jmp_instr_id,
12945                         jmp_instr_id);
12946                 break;
12947
12948         case INSTR_JMP_NEQ_HM:
12949                 fprintf(f,
12950                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12951                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12952                         jmp_instr_id,
12953                         jmp_instr_id);
12954                 break;
12955
12956         case INSTR_JMP_NEQ_HH:
12957                 fprintf(f,
12958                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12959                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12960                         jmp_instr_id,
12961                         jmp_instr_id);
12962                 break;
12963
12964         case INSTR_JMP_NEQ_I:
12965                 fprintf(f,
12966                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12967                         "pipeline_instructions[%u].jmp.b_val)",
12968                         jmp_instr_id,
12969                         jmp_instr_id);
12970                 break;
12971
12972         case INSTR_JMP_LT:
12973                 fprintf(f,
12974                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12975                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12976                         jmp_instr_id,
12977                         jmp_instr_id);
12978                 break;
12979
12980         case INSTR_JMP_LT_MH:
12981                 fprintf(f,
12982                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12983                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12984                         jmp_instr_id,
12985                         jmp_instr_id);
12986                 break;
12987
12988         case INSTR_JMP_LT_HM:
12989                 fprintf(f,
12990                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12991                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12992                         jmp_instr_id,
12993                         jmp_instr_id);
12994                 break;
12995
12996         case INSTR_JMP_LT_HH:
12997                 fprintf(f,
12998                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12999                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13000                         jmp_instr_id,
13001                         jmp_instr_id);
13002                 break;
13003
13004         case INSTR_JMP_LT_MI:
13005                 fprintf(f,
13006                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
13007                         "pipeline_instructions[%u].jmp.b_val)",
13008                         jmp_instr_id,
13009                         jmp_instr_id);
13010                 break;
13011
13012         case INSTR_JMP_LT_HI:
13013                 fprintf(f,
13014                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
13015                         "pipeline_instructions[%u].jmp.b_val)",
13016                         jmp_instr_id,
13017                         jmp_instr_id);
13018                 break;
13019
13020         case INSTR_JMP_GT:
13021                 fprintf(f,
13022                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13023                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
13024                         jmp_instr_id,
13025                         jmp_instr_id);
13026                 break;
13027
13028         case INSTR_JMP_GT_MH:
13029                 fprintf(f,
13030                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13031                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13032                         jmp_instr_id,
13033                         jmp_instr_id);
13034                 break;
13035
13036         case INSTR_JMP_GT_HM:
13037                 fprintf(f,
13038                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13039                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
13040                         jmp_instr_id,
13041                         jmp_instr_id);
13042                 break;
13043
13044         case INSTR_JMP_GT_HH:
13045                 fprintf(f,
13046                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13047                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13048                         jmp_instr_id,
13049                         jmp_instr_id);
13050                 break;
13051
13052         case INSTR_JMP_GT_MI:
13053                 fprintf(f,
13054                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13055                         "pipeline_instructions[%u].jmp.b_val)",
13056                         jmp_instr_id,
13057                         jmp_instr_id);
13058                 break;
13059
13060         case INSTR_JMP_GT_HI:
13061                 fprintf(f,
13062                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13063                         "pipeline_instructions[%u].jmp.b_val)",
13064                         jmp_instr_id,
13065                         jmp_instr_id);
13066                 break;
13067
13068         default:
13069                 break;
13070         }
13071
13072         /* Find the instruction group of the jump instruction. */
13073         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
13074         if (!jmp_g)
13075                 return -EINVAL;
13076
13077         /* Find the instruction group of the jump destination instruction. */
13078         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
13079         if (!data)
13080                 return -EINVAL;
13081
13082         instr_id = data - p->instruction_data;
13083
13084         g = instruction_group_list_group_find(igl, instr_id);
13085         if (!g)
13086                 return -EINVAL;
13087
13088         /* Code generation for "near" jump (same instruction group) or "far" jump (different
13089          * instruction group).
13090          */
13091         if (g->group_id == jmp_g->group_id)
13092                 fprintf(f,
13093                         "\n\t\tgoto %s;\n",
13094                         jmp_data->jmp_label);
13095         else
13096                 fprintf(f,
13097                         " {\n"
13098                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
13099                         "\t\treturn;\n"
13100                         "\t}\n\n",
13101                         g->group_id);
13102
13103         return 0;
13104 }
13105
13106 static void
13107 instruction_group_list_codegen(struct instruction_group_list *igl,
13108                                struct rte_swx_pipeline *p,
13109                                FILE *f)
13110 {
13111         struct instruction_group *g;
13112         uint32_t i;
13113         int is_required = 0;
13114
13115         /* Check if code generation is required. */
13116         TAILQ_FOREACH(g, igl, node)
13117                 if (g->first_instr_id < g->last_instr_id)
13118                         is_required = 1;
13119
13120         if (!is_required)
13121                 return;
13122
13123         /* Generate the code for the pipeline instruction array. */
13124         fprintf(f,
13125                 "static const struct instruction pipeline_instructions[] = {\n");
13126
13127         for (i = 0; i < p->n_instructions; i++) {
13128                 struct instruction *instr = &p->instructions[i];
13129                 instruction_export_t func = export_table[instr->type];
13130
13131                 func(instr, f);
13132         }
13133
13134         fprintf(f, "};\n\n");
13135
13136         /* Generate the code for the pipeline functions: one function for each instruction group
13137          * that contains more than one instruction.
13138          */
13139         TAILQ_FOREACH(g, igl, node) {
13140                 struct instruction *last_instr;
13141                 uint32_t j;
13142
13143                 /* Skip if group contains a single instruction. */
13144                 if (g->last_instr_id == g->first_instr_id)
13145                         continue;
13146
13147                 /* Generate new pipeline function. */
13148                 fprintf(f,
13149                         "void\n"
13150                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
13151                         "{\n"
13152                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
13153                         "\n",
13154                         g->group_id);
13155
13156                 /* Generate the code for each pipeline instruction. */
13157                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
13158                         struct instruction *instr = &p->instructions[j];
13159                         struct instruction_data *data = &p->instruction_data[j];
13160
13161                         /* Label, if present. */
13162                         if (data->label[0])
13163                                 fprintf(f, "\n%s : ", data->label);
13164                         else
13165                                 fprintf(f, "\n\t");
13166
13167                         /* TX instruction type. */
13168                         if (instruction_does_tx(instr)) {
13169                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
13170                                 continue;
13171                         }
13172
13173                         /* Jump instruction type. */
13174                         if (instruction_is_jmp(instr)) {
13175                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
13176                                 continue;
13177                         }
13178
13179                         /* Any other instruction type. */
13180                         fprintf(f,
13181                                 "%s(p, t, &pipeline_instructions[%u]);\n",
13182                                 instr_type_to_func(instr),
13183                                 j);
13184                 }
13185
13186                 /* Finalize the generated pipeline function. For some instructions such as TX,
13187                  * emit-many-and-TX and unconditional jump, the next instruction has been already
13188                  * decided unconditionally and the instruction pointer of the current thread set
13189                  * accordingly; for all the other instructions, the instruction pointer must be
13190                  * incremented now.
13191                  */
13192                 last_instr = &p->instructions[g->last_instr_id];
13193
13194                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
13195                         fprintf(f,
13196                                 "thread_ip_inc(p);\n");
13197
13198                 fprintf(f,
13199                         "}\n"
13200                         "\n");
13201         }
13202 }
13203
13204 static uint32_t
13205 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
13206 {
13207         struct instruction_group *g;
13208         uint32_t n_custom_instr = 0;
13209
13210         /* Groups with a single instruction: no function is generated for this group, the group
13211          * keeps its current instruction. Groups with more than two instructions: one function and
13212          * the associated custom instruction get generated for each such group.
13213          */
13214         TAILQ_FOREACH(g, igl, node) {
13215                 if (g->first_instr_id == g->last_instr_id)
13216                         continue;
13217
13218                 n_custom_instr++;
13219         }
13220
13221         return n_custom_instr;
13222 }
13223
13224 static int
13225 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13226 {
13227         struct action *a;
13228         FILE *f = NULL;
13229
13230         /* Create the .c file. */
13231         f = fopen("/tmp/pipeline.c", "w");
13232         if (!f)
13233                 return -EIO;
13234
13235         /* Include the .h file. */
13236         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
13237
13238         /* Add the code for each action. */
13239         TAILQ_FOREACH(a, &p->actions, node) {
13240                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
13241
13242                 action_data_codegen(a, f);
13243
13244                 fprintf(f, "\n");
13245
13246                 action_instr_codegen(a, f);
13247
13248                 fprintf(f, "\n");
13249         }
13250
13251         /* Add the pipeline code. */
13252         instruction_group_list_codegen(igl, p, f);
13253
13254         /* Close the .c file. */
13255         fclose(f);
13256
13257         return 0;
13258 }
13259
13260 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
13261 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
13262 #endif
13263
13264 static int
13265 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13266 {
13267         struct action *a;
13268         struct instruction_group *g;
13269         char *dir_in, *buffer = NULL;
13270         const char *dir_out;
13271         int status = 0;
13272
13273         /* Get the environment variables. */
13274         dir_in = getenv("RTE_INSTALL_DIR");
13275         if (!dir_in) {
13276                 status = -EINVAL;
13277                 goto free;
13278         }
13279
13280         dir_out = "/tmp";
13281
13282         /* Memory allocation for the command buffer. */
13283         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
13284         if (!buffer) {
13285                 status = -ENOMEM;
13286                 goto free;
13287         }
13288
13289         snprintf(buffer,
13290                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
13291                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
13292                  "-I %s/lib/pipeline "
13293                  "-I %s/lib/eal/include "
13294                  "-I %s/lib/eal/x86/include "
13295                  "-I %s/lib/eal/include/generic "
13296                  "-I %s/lib/meter "
13297                  "-I %s/lib/port "
13298                  "-I %s/lib/table "
13299                  "-I %s/lib/pipeline "
13300                  "-I %s/config "
13301                  "-I %s/build "
13302                  "-I %s/lib/eal/linux/include "
13303                  ">%s/pipeline.log 2>&1 "
13304                  "&& "
13305                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
13306                  ">>%s/pipeline.log 2>&1",
13307                  dir_out,
13308                  dir_out,
13309                  dir_in,
13310                  dir_in,
13311                  dir_in,
13312                  dir_in,
13313                  dir_in,
13314                  dir_in,
13315                  dir_in,
13316                  dir_in,
13317                  dir_in,
13318                  dir_in,
13319                  dir_in,
13320                  dir_out,
13321                  dir_out,
13322                  dir_out,
13323                  dir_out);
13324
13325         /* Build the shared object library. */
13326         status = system(buffer);
13327         if (status)
13328                 goto free;
13329
13330         /* Open library. */
13331         snprintf(buffer,
13332                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
13333                  "%s/libpipeline.so",
13334                  dir_out);
13335
13336         p->lib = dlopen(buffer, RTLD_LAZY);
13337         if (!p->lib) {
13338                 status = -EIO;
13339                 goto free;
13340         }
13341
13342         /* Get the action function symbols. */
13343         TAILQ_FOREACH(a, &p->actions, node) {
13344                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
13345
13346                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
13347                 if (!p->action_funcs[a->id]) {
13348                         status = -EINVAL;
13349                         goto free;
13350                 }
13351         }
13352
13353         /* Get the pipeline function symbols. */
13354         TAILQ_FOREACH(g, igl, node) {
13355                 if (g->first_instr_id == g->last_instr_id)
13356                         continue;
13357
13358                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
13359
13360                 g->func = dlsym(p->lib, buffer);
13361                 if (!g->func) {
13362                         status = -EINVAL;
13363                         goto free;
13364                 }
13365         }
13366
13367 free:
13368         if (status && p->lib) {
13369                 dlclose(p->lib);
13370                 p->lib = NULL;
13371         }
13372
13373         free(buffer);
13374
13375         return status;
13376 }
13377
13378 static int
13379 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
13380                       struct instruction_group_list *igl)
13381 {
13382         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
13383
13384         /* Check that enough space is available within the pipeline instruction table to store all
13385          * the custom instructions.
13386          */
13387         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
13388                 return -ENOSPC;
13389
13390         return 0;
13391 }
13392
13393 static void
13394 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13395 {
13396         struct instruction_group *g;
13397         uint32_t i;
13398
13399         /* Pipeline table instructions. */
13400         for (i = 0; i < p->n_instructions; i++) {
13401                 struct instruction *instr = &p->instructions[i];
13402
13403                 if (instr->type == INSTR_TABLE)
13404                         instr->type = INSTR_TABLE_AF;
13405
13406                 if (instr->type == INSTR_LEARNER)
13407                         instr->type = INSTR_LEARNER_AF;
13408         }
13409
13410         /* Pipeline custom instructions. */
13411         i = 0;
13412         TAILQ_FOREACH(g, igl, node) {
13413                 struct instruction *instr = &p->instructions[g->first_instr_id];
13414                 uint32_t j;
13415
13416                 if (g->first_instr_id == g->last_instr_id)
13417                         continue;
13418
13419                 /* Install a new custom instruction. */
13420                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
13421
13422                 /* First instruction of the group: change its type to the new custom instruction. */
13423                 instr->type = INSTR_CUSTOM_0 + i;
13424
13425                 /* All the subsequent instructions of the group: invalidate. */
13426                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
13427                         struct instruction_data *data = &p->instruction_data[j];
13428
13429                         data->invalid = 1;
13430                 }
13431
13432                 i++;
13433         }
13434
13435         /* Remove the invalidated instructions. */
13436         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
13437
13438         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
13439          * instructions that are the only instruction within their group, so they were left
13440          * unmodified).
13441          */
13442         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
13443 }
13444
13445 static int
13446 pipeline_compile(struct rte_swx_pipeline *p)
13447 {
13448         struct instruction_group_list *igl = NULL;
13449         int status = 0;
13450
13451         igl = instruction_group_list_create(p);
13452         if (!igl) {
13453                 status = -ENOMEM;
13454                 goto free;
13455         }
13456
13457         /* Code generation. */
13458         status = pipeline_codegen(p, igl);
13459         if (status)
13460                 goto free;
13461
13462         /* Build and load the shared object library. */
13463         status = pipeline_libload(p, igl);
13464         if (status)
13465                 goto free;
13466
13467         /* Adjust instructions. */
13468         status = pipeline_adjust_check(p, igl);
13469         if (status)
13470                 goto free;
13471
13472         pipeline_adjust(p, igl);
13473
13474 free:
13475         instruction_group_list_free(igl);
13476
13477         return status;
13478 }