lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <dlfcn.h>
   8
   9 #include <rte_swx_port_ethdev.h>
  10 #include <rte_swx_port_fd.h>
  11 #include <rte_swx_port_ring.h>
  12 #include "rte_swx_port_source_sink.h"
  13
  14 #include <rte_swx_table_em.h>
  15 #include <rte_swx_table_wm.h>
  16
  17 #include "rte_swx_pipeline_internal.h"
  18
  19 #define CHECK(condition, err_code)                                             \
  20 do {                                                                           \
  21         if (!(condition))                                                      \
  22                 return -(err_code);                                            \
  23 } while (0)
  24
  25 #define CHECK_NAME(name, err_code)                                             \
  26         CHECK((name) &&                                                        \
  27               (name)[0] &&                                                     \
  28               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  29               err_code)
  30
  31 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  32         CHECK((instr) &&                                                       \
  33               (instr)[0] &&                                                    \
  34               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  35                RTE_SWX_INSTRUCTION_SIZE),                                      \
  36               err_code)
  37
  38 /*
  39  * Environment.
  40  */
  41 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  42
  43 #include <rte_malloc.h>
  44
  45 static void *
  46 env_malloc(size_t size, size_t alignment, int numa_node)
  47 {
  48         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  49 }
  50
  51 static void
  52 env_free(void *start, size_t size __rte_unused)
  53 {
  54         rte_free(start);
  55 }
  56
  57 #else
  58
  59 #include <numa.h>
  60
  61 static void *
  62 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  63 {
  64         void *start;
  65
  66         if (numa_available() == -1)
  67                 return NULL;
  68
  69         start = numa_alloc_onnode(size, numa_node);
  70         if (!start)
  71                 return NULL;
  72
  73         memset(start, 0, size);
  74         return start;
  75 }
  76
  77 static void
  78 env_free(void *start, size_t size)
  79 {
  80         if (numa_available() == -1)
  81                 return;
  82
  83         numa_free(start, size);
  84 }
  85
  86 #endif
  87
  88 /*
  89  * Struct.
  90  */
  91 static struct struct_type *
  92 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  93 {
  94         struct struct_type *elem;
  95
  96         TAILQ_FOREACH(elem, &p->struct_types, node)
  97                 if (strcmp(elem->name, name) == 0)
  98                         return elem;
  99
 100         return NULL;
 101 }
 102
 103 static struct field *
 104 struct_type_field_find(struct struct_type *st, const char *name)
 105 {
 106         uint32_t i;
 107
 108         for (i = 0; i < st->n_fields; i++) {
 109                 struct field *f = &st->fields[i];
 110
 111                 if (strcmp(f->name, name) == 0)
 112                         return f;
 113         }
 114
 115         return NULL;
 116 }
 117
 118 int
 119 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 120                                       const char *name,
 121                                       struct rte_swx_field_params *fields,
 122                                       uint32_t n_fields,
 123                                       int last_field_has_variable_size)
 124 {
 125         struct struct_type *st;
 126         uint32_t i;
 127
 128         CHECK(p, EINVAL);
 129         CHECK_NAME(name, EINVAL);
 130         CHECK(fields, EINVAL);
 131         CHECK(n_fields, EINVAL);
 132
 133         for (i = 0; i < n_fields; i++) {
 134                 struct rte_swx_field_params *f = &fields[i];
 135                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 136                 uint32_t j;
 137
 138                 CHECK_NAME(f->name, EINVAL);
 139                 CHECK(f->n_bits, EINVAL);
 140                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 141                 CHECK((f->n_bits & 7) == 0, EINVAL);
 142
 143                 for (j = 0; j < i; j++) {
 144                         struct rte_swx_field_params *f_prev = &fields[j];
 145
 146                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 147                 }
 148         }
 149
 150         CHECK(!struct_type_find(p, name), EEXIST);
 151
 152         /* Node allocation. */
 153         st = calloc(1, sizeof(struct struct_type));
 154         CHECK(st, ENOMEM);
 155
 156         st->fields = calloc(n_fields, sizeof(struct field));
 157         if (!st->fields) {
 158                 free(st);
 159                 CHECK(0, ENOMEM);
 160         }
 161
 162         /* Node initialization. */
 163         strcpy(st->name, name);
 164         for (i = 0; i < n_fields; i++) {
 165                 struct field *dst = &st->fields[i];
 166                 struct rte_swx_field_params *src = &fields[i];
 167                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 168
 169                 strcpy(dst->name, src->name);
 170                 dst->n_bits = src->n_bits;
 171                 dst->offset = st->n_bits;
 172                 dst->var_size = var_size;
 173
 174                 st->n_bits += src->n_bits;
 175                 st->n_bits_min += var_size ? 0 : src->n_bits;
 176         }
 177         st->n_fields = n_fields;
 178         st->var_size = last_field_has_variable_size;
 179
 180         /* Node add to tailq. */
 181         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 182
 183         return 0;
 184 }
 185
 186 static int
 187 struct_build(struct rte_swx_pipeline *p)
 188 {
 189         uint32_t i;
 190
 191         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 192                 struct thread *t = &p->threads[i];
 193
 194                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 195                 CHECK(t->structs, ENOMEM);
 196         }
 197
 198         return 0;
 199 }
 200
 201 static void
 202 struct_build_free(struct rte_swx_pipeline *p)
 203 {
 204         uint32_t i;
 205
 206         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 207                 struct thread *t = &p->threads[i];
 208
 209                 free(t->structs);
 210                 t->structs = NULL;
 211         }
 212 }
 213
 214 static void
 215 struct_free(struct rte_swx_pipeline *p)
 216 {
 217         struct_build_free(p);
 218
 219         /* Struct types. */
 220         for ( ; ; ) {
 221                 struct struct_type *elem;
 222
 223                 elem = TAILQ_FIRST(&p->struct_types);
 224                 if (!elem)
 225                         break;
 226
 227                 TAILQ_REMOVE(&p->struct_types, elem, node);
 228                 free(elem->fields);
 229                 free(elem);
 230         }
 231 }
 232
 233 /*
 234  * Input port.
 235  */
 236 static struct port_in_type *
 237 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 238 {
 239         struct port_in_type *elem;
 240
 241         if (!name)
 242                 return NULL;
 243
 244         TAILQ_FOREACH(elem, &p->port_in_types, node)
 245                 if (strcmp(elem->name, name) == 0)
 246                         return elem;
 247
 248         return NULL;
 249 }
 250
 251 int
 252 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 253                                        const char *name,
 254                                        struct rte_swx_port_in_ops *ops)
 255 {
 256         struct port_in_type *elem;
 257
 258         CHECK(p, EINVAL);
 259         CHECK_NAME(name, EINVAL);
 260         CHECK(ops, EINVAL);
 261         CHECK(ops->create, EINVAL);
 262         CHECK(ops->free, EINVAL);
 263         CHECK(ops->pkt_rx, EINVAL);
 264         CHECK(ops->stats_read, EINVAL);
 265
 266         CHECK(!port_in_type_find(p, name), EEXIST);
 267
 268         /* Node allocation. */
 269         elem = calloc(1, sizeof(struct port_in_type));
 270         CHECK(elem, ENOMEM);
 271
 272         /* Node initialization. */
 273         strcpy(elem->name, name);
 274         memcpy(&elem->ops, ops, sizeof(*ops));
 275
 276         /* Node add to tailq. */
 277         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 278
 279         return 0;
 280 }
 281
 282 static struct port_in *
 283 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 284 {
 285         struct port_in *port;
 286
 287         TAILQ_FOREACH(port, &p->ports_in, node)
 288                 if (port->id == port_id)
 289                         return port;
 290
 291         return NULL;
 292 }
 293
 294 int
 295 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 296                                 uint32_t port_id,
 297                                 const char *port_type_name,
 298                                 void *args)
 299 {
 300         struct port_in_type *type = NULL;
 301         struct port_in *port = NULL;
 302         void *obj = NULL;
 303
 304         CHECK(p, EINVAL);
 305
 306         CHECK(!port_in_find(p, port_id), EINVAL);
 307
 308         CHECK_NAME(port_type_name, EINVAL);
 309         type = port_in_type_find(p, port_type_name);
 310         CHECK(type, EINVAL);
 311
 312         obj = type->ops.create(args);
 313         CHECK(obj, ENODEV);
 314
 315         /* Node allocation. */
 316         port = calloc(1, sizeof(struct port_in));
 317         CHECK(port, ENOMEM);
 318
 319         /* Node initialization. */
 320         port->type = type;
 321         port->obj = obj;
 322         port->id = port_id;
 323
 324         /* Node add to tailq. */
 325         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 326         if (p->n_ports_in < port_id + 1)
 327                 p->n_ports_in = port_id + 1;
 328
 329         return 0;
 330 }
 331
 332 static int
 333 port_in_build(struct rte_swx_pipeline *p)
 334 {
 335         struct port_in *port;
 336         uint32_t i;
 337
 338         CHECK(p->n_ports_in, EINVAL);
 339         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 340
 341         for (i = 0; i < p->n_ports_in; i++)
 342                 CHECK(port_in_find(p, i), EINVAL);
 343
 344         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 345         CHECK(p->in, ENOMEM);
 346
 347         TAILQ_FOREACH(port, &p->ports_in, node) {
 348                 struct port_in_runtime *in = &p->in[port->id];
 349
 350                 in->pkt_rx = port->type->ops.pkt_rx;
 351                 in->obj = port->obj;
 352         }
 353
 354         return 0;
 355 }
 356
 357 static void
 358 port_in_build_free(struct rte_swx_pipeline *p)
 359 {
 360         free(p->in);
 361         p->in = NULL;
 362 }
 363
 364 static void
 365 port_in_free(struct rte_swx_pipeline *p)
 366 {
 367         port_in_build_free(p);
 368
 369         /* Input ports. */
 370         for ( ; ; ) {
 371                 struct port_in *port;
 372
 373                 port = TAILQ_FIRST(&p->ports_in);
 374                 if (!port)
 375                         break;
 376
 377                 TAILQ_REMOVE(&p->ports_in, port, node);
 378                 port->type->ops.free(port->obj);
 379                 free(port);
 380         }
 381
 382         /* Input port types. */
 383         for ( ; ; ) {
 384                 struct port_in_type *elem;
 385
 386                 elem = TAILQ_FIRST(&p->port_in_types);
 387                 if (!elem)
 388                         break;
 389
 390                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 391                 free(elem);
 392         }
 393 }
 394
 395 /*
 396  * Output port.
 397  */
 398 static struct port_out_type *
 399 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 400 {
 401         struct port_out_type *elem;
 402
 403         if (!name)
 404                 return NULL;
 405
 406         TAILQ_FOREACH(elem, &p->port_out_types, node)
 407                 if (!strcmp(elem->name, name))
 408                         return elem;
 409
 410         return NULL;
 411 }
 412
 413 int
 414 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 415                                         const char *name,
 416                                         struct rte_swx_port_out_ops *ops)
 417 {
 418         struct port_out_type *elem;
 419
 420         CHECK(p, EINVAL);
 421         CHECK_NAME(name, EINVAL);
 422         CHECK(ops, EINVAL);
 423         CHECK(ops->create, EINVAL);
 424         CHECK(ops->free, EINVAL);
 425         CHECK(ops->pkt_tx, EINVAL);
 426         CHECK(ops->stats_read, EINVAL);
 427
 428         CHECK(!port_out_type_find(p, name), EEXIST);
 429
 430         /* Node allocation. */
 431         elem = calloc(1, sizeof(struct port_out_type));
 432         CHECK(elem, ENOMEM);
 433
 434         /* Node initialization. */
 435         strcpy(elem->name, name);
 436         memcpy(&elem->ops, ops, sizeof(*ops));
 437
 438         /* Node add to tailq. */
 439         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 440
 441         return 0;
 442 }
 443
 444 static struct port_out *
 445 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 446 {
 447         struct port_out *port;
 448
 449         TAILQ_FOREACH(port, &p->ports_out, node)
 450                 if (port->id == port_id)
 451                         return port;
 452
 453         return NULL;
 454 }
 455
 456 int
 457 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 458                                  uint32_t port_id,
 459                                  const char *port_type_name,
 460                                  void *args)
 461 {
 462         struct port_out_type *type = NULL;
 463         struct port_out *port = NULL;
 464         void *obj = NULL;
 465
 466         CHECK(p, EINVAL);
 467
 468         CHECK(!port_out_find(p, port_id), EINVAL);
 469
 470         CHECK_NAME(port_type_name, EINVAL);
 471         type = port_out_type_find(p, port_type_name);
 472         CHECK(type, EINVAL);
 473
 474         obj = type->ops.create(args);
 475         CHECK(obj, ENODEV);
 476
 477         /* Node allocation. */
 478         port = calloc(1, sizeof(struct port_out));
 479         CHECK(port, ENOMEM);
 480
 481         /* Node initialization. */
 482         port->type = type;
 483         port->obj = obj;
 484         port->id = port_id;
 485
 486         /* Node add to tailq. */
 487         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 488         if (p->n_ports_out < port_id + 1)
 489                 p->n_ports_out = port_id + 1;
 490
 491         return 0;
 492 }
 493
 494 static int
 495 port_out_build(struct rte_swx_pipeline *p)
 496 {
 497         struct port_out *port;
 498         uint32_t i;
 499
 500         CHECK(p->n_ports_out, EINVAL);
 501
 502         for (i = 0; i < p->n_ports_out; i++)
 503                 CHECK(port_out_find(p, i), EINVAL);
 504
 505         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 506         CHECK(p->out, ENOMEM);
 507
 508         TAILQ_FOREACH(port, &p->ports_out, node) {
 509                 struct port_out_runtime *out = &p->out[port->id];
 510
 511                 out->pkt_tx = port->type->ops.pkt_tx;
 512                 out->flush = port->type->ops.flush;
 513                 out->obj = port->obj;
 514         }
 515
 516         return 0;
 517 }
 518
 519 static void
 520 port_out_build_free(struct rte_swx_pipeline *p)
 521 {
 522         free(p->out);
 523         p->out = NULL;
 524 }
 525
 526 static void
 527 port_out_free(struct rte_swx_pipeline *p)
 528 {
 529         port_out_build_free(p);
 530
 531         /* Output ports. */
 532         for ( ; ; ) {
 533                 struct port_out *port;
 534
 535                 port = TAILQ_FIRST(&p->ports_out);
 536                 if (!port)
 537                         break;
 538
 539                 TAILQ_REMOVE(&p->ports_out, port, node);
 540                 port->type->ops.free(port->obj);
 541                 free(port);
 542         }
 543
 544         /* Output port types. */
 545         for ( ; ; ) {
 546                 struct port_out_type *elem;
 547
 548                 elem = TAILQ_FIRST(&p->port_out_types);
 549                 if (!elem)
 550                         break;
 551
 552                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 553                 free(elem);
 554         }
 555 }
 556
 557 /*
 558  * Extern object.
 559  */
 560 static struct extern_type *
 561 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 562 {
 563         struct extern_type *elem;
 564
 565         TAILQ_FOREACH(elem, &p->extern_types, node)
 566                 if (strcmp(elem->name, name) == 0)
 567                         return elem;
 568
 569         return NULL;
 570 }
 571
 572 static struct extern_type_member_func *
 573 extern_type_member_func_find(struct extern_type *type, const char *name)
 574 {
 575         struct extern_type_member_func *elem;
 576
 577         TAILQ_FOREACH(elem, &type->funcs, node)
 578                 if (strcmp(elem->name, name) == 0)
 579                         return elem;
 580
 581         return NULL;
 582 }
 583
 584 static struct extern_obj *
 585 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 586 {
 587         struct extern_obj *elem;
 588
 589         TAILQ_FOREACH(elem, &p->extern_objs, node)
 590                 if (strcmp(elem->name, name) == 0)
 591                         return elem;
 592
 593         return NULL;
 594 }
 595
 596 static struct extern_type_member_func *
 597 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 598                              const char *name,
 599                              struct extern_obj **obj)
 600 {
 601         struct extern_obj *object;
 602         struct extern_type_member_func *func;
 603         char *object_name, *func_name;
 604
 605         if (name[0] != 'e' || name[1] != '.')
 606                 return NULL;
 607
 608         object_name = strdup(&name[2]);
 609         if (!object_name)
 610                 return NULL;
 611
 612         func_name = strchr(object_name, '.');
 613         if (!func_name) {
 614                 free(object_name);
 615                 return NULL;
 616         }
 617
 618         *func_name = 0;
 619         func_name++;
 620
 621         object = extern_obj_find(p, object_name);
 622         if (!object) {
 623                 free(object_name);
 624                 return NULL;
 625         }
 626
 627         func = extern_type_member_func_find(object->type, func_name);
 628         if (!func) {
 629                 free(object_name);
 630                 return NULL;
 631         }
 632
 633         if (obj)
 634                 *obj = object;
 635
 636         free(object_name);
 637         return func;
 638 }
 639
 640 static struct field *
 641 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 642                                const char *name,
 643                                struct extern_obj **object)
 644 {
 645         struct extern_obj *obj;
 646         struct field *f;
 647         char *obj_name, *field_name;
 648
 649         if ((name[0] != 'e') || (name[1] != '.'))
 650                 return NULL;
 651
 652         obj_name = strdup(&name[2]);
 653         if (!obj_name)
 654                 return NULL;
 655
 656         field_name = strchr(obj_name, '.');
 657         if (!field_name) {
 658                 free(obj_name);
 659                 return NULL;
 660         }
 661
 662         *field_name = 0;
 663         field_name++;
 664
 665         obj = extern_obj_find(p, obj_name);
 666         if (!obj) {
 667                 free(obj_name);
 668                 return NULL;
 669         }
 670
 671         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 672         if (!f) {
 673                 free(obj_name);
 674                 return NULL;
 675         }
 676
 677         if (object)
 678                 *object = obj;
 679
 680         free(obj_name);
 681         return f;
 682 }
 683
 684 int
 685 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 686         const char *name,
 687         const char *mailbox_struct_type_name,
 688         rte_swx_extern_type_constructor_t constructor,
 689         rte_swx_extern_type_destructor_t destructor)
 690 {
 691         struct extern_type *elem;
 692         struct struct_type *mailbox_struct_type;
 693
 694         CHECK(p, EINVAL);
 695
 696         CHECK_NAME(name, EINVAL);
 697         CHECK(!extern_type_find(p, name), EEXIST);
 698
 699         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 700         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 701         CHECK(mailbox_struct_type, EINVAL);
 702         CHECK(!mailbox_struct_type->var_size, EINVAL);
 703
 704         CHECK(constructor, EINVAL);
 705         CHECK(destructor, EINVAL);
 706
 707         /* Node allocation. */
 708         elem = calloc(1, sizeof(struct extern_type));
 709         CHECK(elem, ENOMEM);
 710
 711         /* Node initialization. */
 712         strcpy(elem->name, name);
 713         elem->mailbox_struct_type = mailbox_struct_type;
 714         elem->constructor = constructor;
 715         elem->destructor = destructor;
 716         TAILQ_INIT(&elem->funcs);
 717
 718         /* Node add to tailq. */
 719         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 720
 721         return 0;
 722 }
 723
 724 int
 725 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 726         const char *extern_type_name,
 727         const char *name,
 728         rte_swx_extern_type_member_func_t member_func)
 729 {
 730         struct extern_type *type;
 731         struct extern_type_member_func *type_member;
 732
 733         CHECK(p, EINVAL);
 734
 735         CHECK_NAME(extern_type_name, EINVAL);
 736         type = extern_type_find(p, extern_type_name);
 737         CHECK(type, EINVAL);
 738         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 739
 740         CHECK_NAME(name, EINVAL);
 741         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 742
 743         CHECK(member_func, EINVAL);
 744
 745         /* Node allocation. */
 746         type_member = calloc(1, sizeof(struct extern_type_member_func));
 747         CHECK(type_member, ENOMEM);
 748
 749         /* Node initialization. */
 750         strcpy(type_member->name, name);
 751         type_member->func = member_func;
 752         type_member->id = type->n_funcs;
 753
 754         /* Node add to tailq. */
 755         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 756         type->n_funcs++;
 757
 758         return 0;
 759 }
 760
 761 int
 762 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 763                                       const char *extern_type_name,
 764                                       const char *name,
 765                                       const char *args)
 766 {
 767         struct extern_type *type;
 768         struct extern_obj *obj;
 769         void *obj_handle;
 770
 771         CHECK(p, EINVAL);
 772
 773         CHECK_NAME(extern_type_name, EINVAL);
 774         type = extern_type_find(p, extern_type_name);
 775         CHECK(type, EINVAL);
 776
 777         CHECK_NAME(name, EINVAL);
 778         CHECK(!extern_obj_find(p, name), EEXIST);
 779
 780         /* Node allocation. */
 781         obj = calloc(1, sizeof(struct extern_obj));
 782         CHECK(obj, ENOMEM);
 783
 784         /* Object construction. */
 785         obj_handle = type->constructor(args);
 786         if (!obj_handle) {
 787                 free(obj);
 788                 CHECK(0, ENODEV);
 789         }
 790
 791         /* Node initialization. */
 792         strcpy(obj->name, name);
 793         obj->type = type;
 794         obj->obj = obj_handle;
 795         obj->struct_id = p->n_structs;
 796         obj->id = p->n_extern_objs;
 797
 798         /* Node add to tailq. */
 799         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 800         p->n_extern_objs++;
 801         p->n_structs++;
 802
 803         return 0;
 804 }
 805
 806 static int
 807 extern_obj_build(struct rte_swx_pipeline *p)
 808 {
 809         uint32_t i;
 810
 811         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 812                 struct thread *t = &p->threads[i];
 813                 struct extern_obj *obj;
 814
 815                 t->extern_objs = calloc(p->n_extern_objs,
 816                                         sizeof(struct extern_obj_runtime));
 817                 CHECK(t->extern_objs, ENOMEM);
 818
 819                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 820                         struct extern_obj_runtime *r =
 821                                 &t->extern_objs[obj->id];
 822                         struct extern_type_member_func *func;
 823                         uint32_t mailbox_size =
 824                                 obj->type->mailbox_struct_type->n_bits / 8;
 825
 826                         r->obj = obj->obj;
 827
 828                         r->mailbox = calloc(1, mailbox_size);
 829                         CHECK(r->mailbox, ENOMEM);
 830
 831                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 832                                 r->funcs[func->id] = func->func;
 833
 834                         t->structs[obj->struct_id] = r->mailbox;
 835                 }
 836         }
 837
 838         return 0;
 839 }
 840
 841 static void
 842 extern_obj_build_free(struct rte_swx_pipeline *p)
 843 {
 844         uint32_t i;
 845
 846         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 847                 struct thread *t = &p->threads[i];
 848                 uint32_t j;
 849
 850                 if (!t->extern_objs)
 851                         continue;
 852
 853                 for (j = 0; j < p->n_extern_objs; j++) {
 854                         struct extern_obj_runtime *r = &t->extern_objs[j];
 855
 856                         free(r->mailbox);
 857                 }
 858
 859                 free(t->extern_objs);
 860                 t->extern_objs = NULL;
 861         }
 862 }
 863
 864 static void
 865 extern_obj_free(struct rte_swx_pipeline *p)
 866 {
 867         extern_obj_build_free(p);
 868
 869         /* Extern objects. */
 870         for ( ; ; ) {
 871                 struct extern_obj *elem;
 872
 873                 elem = TAILQ_FIRST(&p->extern_objs);
 874                 if (!elem)
 875                         break;
 876
 877                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 878                 if (elem->obj)
 879                         elem->type->destructor(elem->obj);
 880                 free(elem);
 881         }
 882
 883         /* Extern types. */
 884         for ( ; ; ) {
 885                 struct extern_type *elem;
 886
 887                 elem = TAILQ_FIRST(&p->extern_types);
 888                 if (!elem)
 889                         break;
 890
 891                 TAILQ_REMOVE(&p->extern_types, elem, node);
 892
 893                 for ( ; ; ) {
 894                         struct extern_type_member_func *func;
 895
 896                         func = TAILQ_FIRST(&elem->funcs);
 897                         if (!func)
 898                                 break;
 899
 900                         TAILQ_REMOVE(&elem->funcs, func, node);
 901                         free(func);
 902                 }
 903
 904                 free(elem);
 905         }
 906 }
 907
 908 /*
 909  * Extern function.
 910  */
 911 static struct extern_func *
 912 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 913 {
 914         struct extern_func *elem;
 915
 916         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 917                 if (strcmp(elem->name, name) == 0)
 918                         return elem;
 919
 920         return NULL;
 921 }
 922
 923 static struct extern_func *
 924 extern_func_parse(struct rte_swx_pipeline *p,
 925                   const char *name)
 926 {
 927         if (name[0] != 'f' || name[1] != '.')
 928                 return NULL;
 929
 930         return extern_func_find(p, &name[2]);
 931 }
 932
 933 static struct field *
 934 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 935                                 const char *name,
 936                                 struct extern_func **function)
 937 {
 938         struct extern_func *func;
 939         struct field *f;
 940         char *func_name, *field_name;
 941
 942         if ((name[0] != 'f') || (name[1] != '.'))
 943                 return NULL;
 944
 945         func_name = strdup(&name[2]);
 946         if (!func_name)
 947                 return NULL;
 948
 949         field_name = strchr(func_name, '.');
 950         if (!field_name) {
 951                 free(func_name);
 952                 return NULL;
 953         }
 954
 955         *field_name = 0;
 956         field_name++;
 957
 958         func = extern_func_find(p, func_name);
 959         if (!func) {
 960                 free(func_name);
 961                 return NULL;
 962         }
 963
 964         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 965         if (!f) {
 966                 free(func_name);
 967                 return NULL;
 968         }
 969
 970         if (function)
 971                 *function = func;
 972
 973         free(func_name);
 974         return f;
 975 }
 976
 977 int
 978 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 979                                       const char *name,
 980                                       const char *mailbox_struct_type_name,
 981                                       rte_swx_extern_func_t func)
 982 {
 983         struct extern_func *f;
 984         struct struct_type *mailbox_struct_type;
 985
 986         CHECK(p, EINVAL);
 987
 988         CHECK_NAME(name, EINVAL);
 989         CHECK(!extern_func_find(p, name), EEXIST);
 990
 991         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 992         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 993         CHECK(mailbox_struct_type, EINVAL);
 994         CHECK(!mailbox_struct_type->var_size, EINVAL);
 995
 996         CHECK(func, EINVAL);
 997
 998         /* Node allocation. */
 999         f = calloc(1, sizeof(struct extern_func));
1000         CHECK(func, ENOMEM);
1001
1002         /* Node initialization. */
1003         strcpy(f->name, name);
1004         f->mailbox_struct_type = mailbox_struct_type;
1005         f->func = func;
1006         f->struct_id = p->n_structs;
1007         f->id = p->n_extern_funcs;
1008
1009         /* Node add to tailq. */
1010         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1011         p->n_extern_funcs++;
1012         p->n_structs++;
1013
1014         return 0;
1015 }
1016
1017 static int
1018 extern_func_build(struct rte_swx_pipeline *p)
1019 {
1020         uint32_t i;
1021
1022         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1023                 struct thread *t = &p->threads[i];
1024                 struct extern_func *func;
1025
1026                 /* Memory allocation. */
1027                 t->extern_funcs = calloc(p->n_extern_funcs,
1028                                          sizeof(struct extern_func_runtime));
1029                 CHECK(t->extern_funcs, ENOMEM);
1030
1031                 /* Extern function. */
1032                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1033                         struct extern_func_runtime *r =
1034                                 &t->extern_funcs[func->id];
1035                         uint32_t mailbox_size =
1036                                 func->mailbox_struct_type->n_bits / 8;
1037
1038                         r->func = func->func;
1039
1040                         r->mailbox = calloc(1, mailbox_size);
1041                         CHECK(r->mailbox, ENOMEM);
1042
1043                         t->structs[func->struct_id] = r->mailbox;
1044                 }
1045         }
1046
1047         return 0;
1048 }
1049
1050 static void
1051 extern_func_build_free(struct rte_swx_pipeline *p)
1052 {
1053         uint32_t i;
1054
1055         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1056                 struct thread *t = &p->threads[i];
1057                 uint32_t j;
1058
1059                 if (!t->extern_funcs)
1060                         continue;
1061
1062                 for (j = 0; j < p->n_extern_funcs; j++) {
1063                         struct extern_func_runtime *r = &t->extern_funcs[j];
1064
1065                         free(r->mailbox);
1066                 }
1067
1068                 free(t->extern_funcs);
1069                 t->extern_funcs = NULL;
1070         }
1071 }
1072
1073 static void
1074 extern_func_free(struct rte_swx_pipeline *p)
1075 {
1076         extern_func_build_free(p);
1077
1078         for ( ; ; ) {
1079                 struct extern_func *elem;
1080
1081                 elem = TAILQ_FIRST(&p->extern_funcs);
1082                 if (!elem)
1083                         break;
1084
1085                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1086                 free(elem);
1087         }
1088 }
1089
1090 /*
1091  * Header.
1092  */
1093 static struct header *
1094 header_find(struct rte_swx_pipeline *p, const char *name)
1095 {
1096         struct header *elem;
1097
1098         TAILQ_FOREACH(elem, &p->headers, node)
1099                 if (strcmp(elem->name, name) == 0)
1100                         return elem;
1101
1102         return NULL;
1103 }
1104
1105 static struct header *
1106 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1107 {
1108         struct header *elem;
1109
1110         TAILQ_FOREACH(elem, &p->headers, node)
1111                 if (elem->struct_id == struct_id)
1112                         return elem;
1113
1114         return NULL;
1115 }
1116
1117 static struct header *
1118 header_parse(struct rte_swx_pipeline *p,
1119              const char *name)
1120 {
1121         if (name[0] != 'h' || name[1] != '.')
1122                 return NULL;
1123
1124         return header_find(p, &name[2]);
1125 }
1126
1127 static struct field *
1128 header_field_parse(struct rte_swx_pipeline *p,
1129                    const char *name,
1130                    struct header **header)
1131 {
1132         struct header *h;
1133         struct field *f;
1134         char *header_name, *field_name;
1135
1136         if ((name[0] != 'h') || (name[1] != '.'))
1137                 return NULL;
1138
1139         header_name = strdup(&name[2]);
1140         if (!header_name)
1141                 return NULL;
1142
1143         field_name = strchr(header_name, '.');
1144         if (!field_name) {
1145                 free(header_name);
1146                 return NULL;
1147         }
1148
1149         *field_name = 0;
1150         field_name++;
1151
1152         h = header_find(p, header_name);
1153         if (!h) {
1154                 free(header_name);
1155                 return NULL;
1156         }
1157
1158         f = struct_type_field_find(h->st, field_name);
1159         if (!f) {
1160                 free(header_name);
1161                 return NULL;
1162         }
1163
1164         if (header)
1165                 *header = h;
1166
1167         free(header_name);
1168         return f;
1169 }
1170
1171 int
1172 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1173                                         const char *name,
1174                                         const char *struct_type_name)
1175 {
1176         struct struct_type *st;
1177         struct header *h;
1178         size_t n_headers_max;
1179
1180         CHECK(p, EINVAL);
1181         CHECK_NAME(name, EINVAL);
1182         CHECK_NAME(struct_type_name, EINVAL);
1183
1184         CHECK(!header_find(p, name), EEXIST);
1185
1186         st = struct_type_find(p, struct_type_name);
1187         CHECK(st, EINVAL);
1188
1189         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1190         CHECK(p->n_headers < n_headers_max, ENOSPC);
1191
1192         /* Node allocation. */
1193         h = calloc(1, sizeof(struct header));
1194         CHECK(h, ENOMEM);
1195
1196         /* Node initialization. */
1197         strcpy(h->name, name);
1198         h->st = st;
1199         h->struct_id = p->n_structs;
1200         h->id = p->n_headers;
1201
1202         /* Node add to tailq. */
1203         TAILQ_INSERT_TAIL(&p->headers, h, node);
1204         p->n_headers++;
1205         p->n_structs++;
1206
1207         return 0;
1208 }
1209
1210 static int
1211 header_build(struct rte_swx_pipeline *p)
1212 {
1213         struct header *h;
1214         uint32_t n_bytes = 0, i;
1215
1216         TAILQ_FOREACH(h, &p->headers, node) {
1217                 n_bytes += h->st->n_bits / 8;
1218         }
1219
1220         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1221                 struct thread *t = &p->threads[i];
1222                 uint32_t offset = 0;
1223
1224                 t->headers = calloc(p->n_headers,
1225                                     sizeof(struct header_runtime));
1226                 CHECK(t->headers, ENOMEM);
1227
1228                 t->headers_out = calloc(p->n_headers,
1229                                         sizeof(struct header_out_runtime));
1230                 CHECK(t->headers_out, ENOMEM);
1231
1232                 t->header_storage = calloc(1, n_bytes);
1233                 CHECK(t->header_storage, ENOMEM);
1234
1235                 t->header_out_storage = calloc(1, n_bytes);
1236                 CHECK(t->header_out_storage, ENOMEM);
1237
1238                 TAILQ_FOREACH(h, &p->headers, node) {
1239                         uint8_t *header_storage;
1240                         uint32_t n_bytes =  h->st->n_bits / 8;
1241
1242                         header_storage = &t->header_storage[offset];
1243                         offset += n_bytes;
1244
1245                         t->headers[h->id].ptr0 = header_storage;
1246                         t->headers[h->id].n_bytes = n_bytes;
1247
1248                         t->structs[h->struct_id] = header_storage;
1249                 }
1250         }
1251
1252         return 0;
1253 }
1254
1255 static void
1256 header_build_free(struct rte_swx_pipeline *p)
1257 {
1258         uint32_t i;
1259
1260         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1261                 struct thread *t = &p->threads[i];
1262
1263                 free(t->headers_out);
1264                 t->headers_out = NULL;
1265
1266                 free(t->headers);
1267                 t->headers = NULL;
1268
1269                 free(t->header_out_storage);
1270                 t->header_out_storage = NULL;
1271
1272                 free(t->header_storage);
1273                 t->header_storage = NULL;
1274         }
1275 }
1276
1277 static void
1278 header_free(struct rte_swx_pipeline *p)
1279 {
1280         header_build_free(p);
1281
1282         for ( ; ; ) {
1283                 struct header *elem;
1284
1285                 elem = TAILQ_FIRST(&p->headers);
1286                 if (!elem)
1287                         break;
1288
1289                 TAILQ_REMOVE(&p->headers, elem, node);
1290                 free(elem);
1291         }
1292 }
1293
1294 /*
1295  * Meta-data.
1296  */
1297 static struct field *
1298 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1299 {
1300         if (!p->metadata_st)
1301                 return NULL;
1302
1303         if (name[0] != 'm' || name[1] != '.')
1304                 return NULL;
1305
1306         return struct_type_field_find(p->metadata_st, &name[2]);
1307 }
1308
1309 int
1310 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1311                                           const char *struct_type_name)
1312 {
1313         struct struct_type *st = NULL;
1314
1315         CHECK(p, EINVAL);
1316
1317         CHECK_NAME(struct_type_name, EINVAL);
1318         st  = struct_type_find(p, struct_type_name);
1319         CHECK(st, EINVAL);
1320         CHECK(!st->var_size, EINVAL);
1321         CHECK(!p->metadata_st, EINVAL);
1322
1323         p->metadata_st = st;
1324         p->metadata_struct_id = p->n_structs;
1325
1326         p->n_structs++;
1327
1328         return 0;
1329 }
1330
1331 static int
1332 metadata_build(struct rte_swx_pipeline *p)
1333 {
1334         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1335         uint32_t i;
1336
1337         /* Thread-level initialization. */
1338         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1339                 struct thread *t = &p->threads[i];
1340                 uint8_t *metadata;
1341
1342                 metadata = calloc(1, n_bytes);
1343                 CHECK(metadata, ENOMEM);
1344
1345                 t->metadata = metadata;
1346                 t->structs[p->metadata_struct_id] = metadata;
1347         }
1348
1349         return 0;
1350 }
1351
1352 static void
1353 metadata_build_free(struct rte_swx_pipeline *p)
1354 {
1355         uint32_t i;
1356
1357         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1358                 struct thread *t = &p->threads[i];
1359
1360                 free(t->metadata);
1361                 t->metadata = NULL;
1362         }
1363 }
1364
1365 static void
1366 metadata_free(struct rte_swx_pipeline *p)
1367 {
1368         metadata_build_free(p);
1369 }
1370
1371 /*
1372  * Instruction.
1373  */
1374 static int
1375 instruction_is_tx(enum instruction_type type)
1376 {
1377         switch (type) {
1378         case INSTR_TX:
1379         case INSTR_TX_I:
1380         case INSTR_DROP:
1381                 return 1;
1382
1383         default:
1384                 return 0;
1385         }
1386 }
1387
1388 static int
1389 instruction_does_tx(struct instruction *instr)
1390 {
1391         switch (instr->type) {
1392         case INSTR_TX:
1393         case INSTR_TX_I:
1394         case INSTR_DROP:
1395         case INSTR_HDR_EMIT_TX:
1396         case INSTR_HDR_EMIT2_TX:
1397         case INSTR_HDR_EMIT3_TX:
1398         case INSTR_HDR_EMIT4_TX:
1399         case INSTR_HDR_EMIT5_TX:
1400         case INSTR_HDR_EMIT6_TX:
1401         case INSTR_HDR_EMIT7_TX:
1402         case INSTR_HDR_EMIT8_TX:
1403                 return 1;
1404         default:
1405                 return 0;
1406         }
1407 }
1408
1409 static int
1410 instruction_is_jmp(struct instruction *instr)
1411 {
1412         switch (instr->type) {
1413         case INSTR_JMP:
1414         case INSTR_JMP_VALID:
1415         case INSTR_JMP_INVALID:
1416         case INSTR_JMP_HIT:
1417         case INSTR_JMP_MISS:
1418         case INSTR_JMP_ACTION_HIT:
1419         case INSTR_JMP_ACTION_MISS:
1420         case INSTR_JMP_EQ:
1421         case INSTR_JMP_EQ_MH:
1422         case INSTR_JMP_EQ_HM:
1423         case INSTR_JMP_EQ_HH:
1424         case INSTR_JMP_EQ_I:
1425         case INSTR_JMP_NEQ:
1426         case INSTR_JMP_NEQ_MH:
1427         case INSTR_JMP_NEQ_HM:
1428         case INSTR_JMP_NEQ_HH:
1429         case INSTR_JMP_NEQ_I:
1430         case INSTR_JMP_LT:
1431         case INSTR_JMP_LT_MH:
1432         case INSTR_JMP_LT_HM:
1433         case INSTR_JMP_LT_HH:
1434         case INSTR_JMP_LT_MI:
1435         case INSTR_JMP_LT_HI:
1436         case INSTR_JMP_GT:
1437         case INSTR_JMP_GT_MH:
1438         case INSTR_JMP_GT_HM:
1439         case INSTR_JMP_GT_HH:
1440         case INSTR_JMP_GT_MI:
1441         case INSTR_JMP_GT_HI:
1442                 return 1;
1443
1444         default:
1445                 return 0;
1446         }
1447 }
1448
1449 static int
1450 instruction_does_thread_yield(struct instruction *instr)
1451 {
1452         switch (instr->type) {
1453         case INSTR_RX:
1454         case INSTR_TABLE:
1455         case INSTR_TABLE_AF:
1456         case INSTR_SELECTOR:
1457         case INSTR_LEARNER:
1458         case INSTR_LEARNER_AF:
1459         case INSTR_EXTERN_OBJ:
1460         case INSTR_EXTERN_FUNC:
1461                 return 1;
1462         default:
1463                 return 0;
1464         }
1465 }
1466
1467 static struct field *
1468 action_field_parse(struct action *action, const char *name);
1469
1470 static struct field *
1471 struct_field_parse(struct rte_swx_pipeline *p,
1472                    struct action *action,
1473                    const char *name,
1474                    uint32_t *struct_id)
1475 {
1476         struct field *f;
1477
1478         switch (name[0]) {
1479         case 'h':
1480         {
1481                 struct header *header;
1482
1483                 f = header_field_parse(p, name, &header);
1484                 if (!f)
1485                         return NULL;
1486
1487                 *struct_id = header->struct_id;
1488                 return f;
1489         }
1490
1491         case 'm':
1492         {
1493                 f = metadata_field_parse(p, name);
1494                 if (!f)
1495                         return NULL;
1496
1497                 *struct_id = p->metadata_struct_id;
1498                 return f;
1499         }
1500
1501         case 't':
1502         {
1503                 if (!action)
1504                         return NULL;
1505
1506                 f = action_field_parse(action, name);
1507                 if (!f)
1508                         return NULL;
1509
1510                 *struct_id = 0;
1511                 return f;
1512         }
1513
1514         case 'e':
1515         {
1516                 struct extern_obj *obj;
1517
1518                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1519                 if (!f)
1520                         return NULL;
1521
1522                 *struct_id = obj->struct_id;
1523                 return f;
1524         }
1525
1526         case 'f':
1527         {
1528                 struct extern_func *func;
1529
1530                 f = extern_func_mailbox_field_parse(p, name, &func);
1531                 if (!f)
1532                         return NULL;
1533
1534                 *struct_id = func->struct_id;
1535                 return f;
1536         }
1537
1538         default:
1539                 return NULL;
1540         }
1541 }
1542
1543 /*
1544  * rx.
1545  */
1546 static int
1547 instr_rx_translate(struct rte_swx_pipeline *p,
1548                    struct action *action,
1549                    char **tokens,
1550                    int n_tokens,
1551                    struct instruction *instr,
1552                    struct instruction_data *data __rte_unused)
1553 {
1554         struct field *f;
1555
1556         CHECK(!action, EINVAL);
1557         CHECK(n_tokens == 2, EINVAL);
1558
1559         f = metadata_field_parse(p, tokens[1]);
1560         CHECK(f, EINVAL);
1561
1562         instr->type = INSTR_RX;
1563         instr->io.io.offset = f->offset / 8;
1564         instr->io.io.n_bits = f->n_bits;
1565         return 0;
1566 }
1567
1568 /*
1569  * tx.
1570  */
1571 static int
1572 instr_tx_translate(struct rte_swx_pipeline *p,
1573                    struct action *action __rte_unused,
1574                    char **tokens,
1575                    int n_tokens,
1576                    struct instruction *instr,
1577                    struct instruction_data *data __rte_unused)
1578 {
1579         char *port = tokens[1];
1580         struct field *f;
1581         uint32_t port_val;
1582
1583         CHECK(n_tokens == 2, EINVAL);
1584
1585         f = metadata_field_parse(p, port);
1586         if (f) {
1587                 instr->type = INSTR_TX;
1588                 instr->io.io.offset = f->offset / 8;
1589                 instr->io.io.n_bits = f->n_bits;
1590                 return 0;
1591         }
1592
1593         /* TX_I. */
1594         port_val = strtoul(port, &port, 0);
1595         CHECK(!port[0], EINVAL);
1596
1597         instr->type = INSTR_TX_I;
1598         instr->io.io.val = port_val;
1599         return 0;
1600 }
1601
1602 static int
1603 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1604                      struct action *action __rte_unused,
1605                      char **tokens __rte_unused,
1606                      int n_tokens,
1607                      struct instruction *instr,
1608                      struct instruction_data *data __rte_unused)
1609 {
1610         CHECK(n_tokens == 1, EINVAL);
1611
1612         /* DROP. */
1613         instr->type = INSTR_DROP;
1614         return 0;
1615 }
1616
1617 static inline void
1618 instr_tx_exec(struct rte_swx_pipeline *p)
1619 {
1620         struct thread *t = &p->threads[p->thread_id];
1621         struct instruction *ip = t->ip;
1622
1623         __instr_tx_exec(p, t, ip);
1624
1625         /* Thread. */
1626         thread_ip_reset(p, t);
1627         instr_rx_exec(p);
1628 }
1629
1630 static inline void
1631 instr_tx_i_exec(struct rte_swx_pipeline *p)
1632 {
1633         struct thread *t = &p->threads[p->thread_id];
1634         struct instruction *ip = t->ip;
1635
1636         __instr_tx_i_exec(p, t, ip);
1637
1638         /* Thread. */
1639         thread_ip_reset(p, t);
1640         instr_rx_exec(p);
1641 }
1642
1643 static inline void
1644 instr_drop_exec(struct rte_swx_pipeline *p)
1645 {
1646         struct thread *t = &p->threads[p->thread_id];
1647         struct instruction *ip = t->ip;
1648
1649         __instr_drop_exec(p, t, ip);
1650
1651         /* Thread. */
1652         thread_ip_reset(p, t);
1653         instr_rx_exec(p);
1654 }
1655
1656 /*
1657  * extract.
1658  */
1659 static int
1660 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1661                             struct action *action,
1662                             char **tokens,
1663                             int n_tokens,
1664                             struct instruction *instr,
1665                             struct instruction_data *data __rte_unused)
1666 {
1667         struct header *h;
1668
1669         CHECK(!action, EINVAL);
1670         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1671
1672         h = header_parse(p, tokens[1]);
1673         CHECK(h, EINVAL);
1674
1675         if (n_tokens == 2) {
1676                 CHECK(!h->st->var_size, EINVAL);
1677
1678                 instr->type = INSTR_HDR_EXTRACT;
1679                 instr->io.hdr.header_id[0] = h->id;
1680                 instr->io.hdr.struct_id[0] = h->struct_id;
1681                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1682         } else {
1683                 struct field *mf;
1684
1685                 CHECK(h->st->var_size, EINVAL);
1686
1687                 mf = metadata_field_parse(p, tokens[2]);
1688                 CHECK(mf, EINVAL);
1689                 CHECK(!mf->var_size, EINVAL);
1690
1691                 instr->type = INSTR_HDR_EXTRACT_M;
1692                 instr->io.io.offset = mf->offset / 8;
1693                 instr->io.io.n_bits = mf->n_bits;
1694                 instr->io.hdr.header_id[0] = h->id;
1695                 instr->io.hdr.struct_id[0] = h->struct_id;
1696                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1697         }
1698
1699         return 0;
1700 }
1701
1702 static int
1703 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1704                               struct action *action,
1705                               char **tokens,
1706                               int n_tokens,
1707                               struct instruction *instr,
1708                               struct instruction_data *data __rte_unused)
1709 {
1710         struct header *h;
1711
1712         CHECK(!action, EINVAL);
1713         CHECK(n_tokens == 2, EINVAL);
1714
1715         h = header_parse(p, tokens[1]);
1716         CHECK(h, EINVAL);
1717         CHECK(!h->st->var_size, EINVAL);
1718
1719         instr->type = INSTR_HDR_LOOKAHEAD;
1720         instr->io.hdr.header_id[0] = h->id;
1721         instr->io.hdr.struct_id[0] = h->struct_id;
1722         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1723
1724         return 0;
1725 }
1726
1727 static inline void
1728 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1729 {
1730         struct thread *t = &p->threads[p->thread_id];
1731         struct instruction *ip = t->ip;
1732
1733         __instr_hdr_extract_exec(p, t, ip);
1734
1735         /* Thread. */
1736         thread_ip_inc(p);
1737 }
1738
1739 static inline void
1740 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1741 {
1742         struct thread *t = &p->threads[p->thread_id];
1743         struct instruction *ip = t->ip;
1744
1745         __instr_hdr_extract2_exec(p, t, ip);
1746
1747         /* Thread. */
1748         thread_ip_inc(p);
1749 }
1750
1751 static inline void
1752 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1753 {
1754         struct thread *t = &p->threads[p->thread_id];
1755         struct instruction *ip = t->ip;
1756
1757         __instr_hdr_extract3_exec(p, t, ip);
1758
1759         /* Thread. */
1760         thread_ip_inc(p);
1761 }
1762
1763 static inline void
1764 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1765 {
1766         struct thread *t = &p->threads[p->thread_id];
1767         struct instruction *ip = t->ip;
1768
1769         __instr_hdr_extract4_exec(p, t, ip);
1770
1771         /* Thread. */
1772         thread_ip_inc(p);
1773 }
1774
1775 static inline void
1776 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1777 {
1778         struct thread *t = &p->threads[p->thread_id];
1779         struct instruction *ip = t->ip;
1780
1781         __instr_hdr_extract5_exec(p, t, ip);
1782
1783         /* Thread. */
1784         thread_ip_inc(p);
1785 }
1786
1787 static inline void
1788 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1789 {
1790         struct thread *t = &p->threads[p->thread_id];
1791         struct instruction *ip = t->ip;
1792
1793         __instr_hdr_extract6_exec(p, t, ip);
1794
1795         /* Thread. */
1796         thread_ip_inc(p);
1797 }
1798
1799 static inline void
1800 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1801 {
1802         struct thread *t = &p->threads[p->thread_id];
1803         struct instruction *ip = t->ip;
1804
1805         __instr_hdr_extract7_exec(p, t, ip);
1806
1807         /* Thread. */
1808         thread_ip_inc(p);
1809 }
1810
1811 static inline void
1812 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1813 {
1814         struct thread *t = &p->threads[p->thread_id];
1815         struct instruction *ip = t->ip;
1816
1817         __instr_hdr_extract8_exec(p, t, ip);
1818
1819         /* Thread. */
1820         thread_ip_inc(p);
1821 }
1822
1823 static inline void
1824 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1825 {
1826         struct thread *t = &p->threads[p->thread_id];
1827         struct instruction *ip = t->ip;
1828
1829         __instr_hdr_extract_m_exec(p, t, ip);
1830
1831         /* Thread. */
1832         thread_ip_inc(p);
1833 }
1834
1835 static inline void
1836 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1837 {
1838         struct thread *t = &p->threads[p->thread_id];
1839         struct instruction *ip = t->ip;
1840
1841         __instr_hdr_lookahead_exec(p, t, ip);
1842
1843         /* Thread. */
1844         thread_ip_inc(p);
1845 }
1846
1847 /*
1848  * emit.
1849  */
1850 static int
1851 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1852                          struct action *action __rte_unused,
1853                          char **tokens,
1854                          int n_tokens,
1855                          struct instruction *instr,
1856                          struct instruction_data *data __rte_unused)
1857 {
1858         struct header *h;
1859
1860         CHECK(n_tokens == 2, EINVAL);
1861
1862         h = header_parse(p, tokens[1]);
1863         CHECK(h, EINVAL);
1864
1865         instr->type = INSTR_HDR_EMIT;
1866         instr->io.hdr.header_id[0] = h->id;
1867         instr->io.hdr.struct_id[0] = h->struct_id;
1868         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1869         return 0;
1870 }
1871
1872 static inline void
1873 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1874 {
1875         struct thread *t = &p->threads[p->thread_id];
1876         struct instruction *ip = t->ip;
1877
1878         __instr_hdr_emit_exec(p, t, ip);
1879
1880         /* Thread. */
1881         thread_ip_inc(p);
1882 }
1883
1884 static inline void
1885 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1886 {
1887         struct thread *t = &p->threads[p->thread_id];
1888         struct instruction *ip = t->ip;
1889
1890         __instr_hdr_emit_tx_exec(p, t, ip);
1891
1892         /* Thread. */
1893         thread_ip_reset(p, t);
1894         instr_rx_exec(p);
1895 }
1896
1897 static inline void
1898 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1899 {
1900         struct thread *t = &p->threads[p->thread_id];
1901         struct instruction *ip = t->ip;
1902
1903         __instr_hdr_emit2_tx_exec(p, t, ip);
1904
1905         /* Thread. */
1906         thread_ip_reset(p, t);
1907         instr_rx_exec(p);
1908 }
1909
1910 static inline void
1911 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1912 {
1913         struct thread *t = &p->threads[p->thread_id];
1914         struct instruction *ip = t->ip;
1915
1916         __instr_hdr_emit3_tx_exec(p, t, ip);
1917
1918         /* Thread. */
1919         thread_ip_reset(p, t);
1920         instr_rx_exec(p);
1921 }
1922
1923 static inline void
1924 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1925 {
1926         struct thread *t = &p->threads[p->thread_id];
1927         struct instruction *ip = t->ip;
1928
1929         __instr_hdr_emit4_tx_exec(p, t, ip);
1930
1931         /* Thread. */
1932         thread_ip_reset(p, t);
1933         instr_rx_exec(p);
1934 }
1935
1936 static inline void
1937 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1938 {
1939         struct thread *t = &p->threads[p->thread_id];
1940         struct instruction *ip = t->ip;
1941
1942         __instr_hdr_emit5_tx_exec(p, t, ip);
1943
1944         /* Thread. */
1945         thread_ip_reset(p, t);
1946         instr_rx_exec(p);
1947 }
1948
1949 static inline void
1950 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1951 {
1952         struct thread *t = &p->threads[p->thread_id];
1953         struct instruction *ip = t->ip;
1954
1955         __instr_hdr_emit6_tx_exec(p, t, ip);
1956
1957         /* Thread. */
1958         thread_ip_reset(p, t);
1959         instr_rx_exec(p);
1960 }
1961
1962 static inline void
1963 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1964 {
1965         struct thread *t = &p->threads[p->thread_id];
1966         struct instruction *ip = t->ip;
1967
1968         __instr_hdr_emit7_tx_exec(p, t, ip);
1969
1970         /* Thread. */
1971         thread_ip_reset(p, t);
1972         instr_rx_exec(p);
1973 }
1974
1975 static inline void
1976 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1977 {
1978         struct thread *t = &p->threads[p->thread_id];
1979         struct instruction *ip = t->ip;
1980
1981         __instr_hdr_emit8_tx_exec(p, t, ip);
1982
1983         /* Thread. */
1984         thread_ip_reset(p, t);
1985         instr_rx_exec(p);
1986 }
1987
1988 /*
1989  * validate.
1990  */
1991 static int
1992 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1993                              struct action *action __rte_unused,
1994                              char **tokens,
1995                              int n_tokens,
1996                              struct instruction *instr,
1997                              struct instruction_data *data __rte_unused)
1998 {
1999         struct header *h;
2000
2001         CHECK(n_tokens == 2, EINVAL);
2002
2003         h = header_parse(p, tokens[1]);
2004         CHECK(h, EINVAL);
2005
2006         instr->type = INSTR_HDR_VALIDATE;
2007         instr->valid.header_id = h->id;
2008         return 0;
2009 }
2010
2011 static inline void
2012 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2013 {
2014         struct thread *t = &p->threads[p->thread_id];
2015         struct instruction *ip = t->ip;
2016
2017         __instr_hdr_validate_exec(p, t, ip);
2018
2019         /* Thread. */
2020         thread_ip_inc(p);
2021 }
2022
2023 /*
2024  * invalidate.
2025  */
2026 static int
2027 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2028                                struct action *action __rte_unused,
2029                                char **tokens,
2030                                int n_tokens,
2031                                struct instruction *instr,
2032                                struct instruction_data *data __rte_unused)
2033 {
2034         struct header *h;
2035
2036         CHECK(n_tokens == 2, EINVAL);
2037
2038         h = header_parse(p, tokens[1]);
2039         CHECK(h, EINVAL);
2040
2041         instr->type = INSTR_HDR_INVALIDATE;
2042         instr->valid.header_id = h->id;
2043         return 0;
2044 }
2045
2046 static inline void
2047 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2048 {
2049         struct thread *t = &p->threads[p->thread_id];
2050         struct instruction *ip = t->ip;
2051
2052         __instr_hdr_invalidate_exec(p, t, ip);
2053
2054         /* Thread. */
2055         thread_ip_inc(p);
2056 }
2057
2058 /*
2059  * table.
2060  */
2061 static struct table *
2062 table_find(struct rte_swx_pipeline *p, const char *name);
2063
2064 static struct selector *
2065 selector_find(struct rte_swx_pipeline *p, const char *name);
2066
2067 static struct learner *
2068 learner_find(struct rte_swx_pipeline *p, const char *name);
2069
2070 static int
2071 instr_table_translate(struct rte_swx_pipeline *p,
2072                       struct action *action,
2073                       char **tokens,
2074                       int n_tokens,
2075                       struct instruction *instr,
2076                       struct instruction_data *data __rte_unused)
2077 {
2078         struct table *t;
2079         struct selector *s;
2080         struct learner *l;
2081
2082         CHECK(!action, EINVAL);
2083         CHECK(n_tokens == 2, EINVAL);
2084
2085         t = table_find(p, tokens[1]);
2086         if (t) {
2087                 instr->type = INSTR_TABLE;
2088                 instr->table.table_id = t->id;
2089                 return 0;
2090         }
2091
2092         s = selector_find(p, tokens[1]);
2093         if (s) {
2094                 instr->type = INSTR_SELECTOR;
2095                 instr->table.table_id = s->id;
2096                 return 0;
2097         }
2098
2099         l = learner_find(p, tokens[1]);
2100         if (l) {
2101                 instr->type = INSTR_LEARNER;
2102                 instr->table.table_id = l->id;
2103                 return 0;
2104         }
2105
2106         CHECK(0, EINVAL);
2107 }
2108
2109 static inline void
2110 instr_table_exec(struct rte_swx_pipeline *p)
2111 {
2112         struct thread *t = &p->threads[p->thread_id];
2113         struct instruction *ip = t->ip;
2114         uint32_t table_id = ip->table.table_id;
2115         struct rte_swx_table_state *ts = &t->table_state[table_id];
2116         struct table_runtime *table = &t->tables[table_id];
2117         struct table_statistics *stats = &p->table_stats[table_id];
2118         uint64_t action_id, n_pkts_hit, n_pkts_action;
2119         uint8_t *action_data;
2120         int done, hit;
2121
2122         /* Table. */
2123         done = table->func(ts->obj,
2124                            table->mailbox,
2125                            table->key,
2126                            &action_id,
2127                            &action_data,
2128                            &hit);
2129         if (!done) {
2130                 /* Thread. */
2131                 TRACE("[Thread %2u] table %u (not finalized)\n",
2132                       p->thread_id,
2133                       table_id);
2134
2135                 thread_yield(p);
2136                 return;
2137         }
2138
2139         action_id = hit ? action_id : ts->default_action_id;
2140         action_data = hit ? action_data : ts->default_action_data;
2141         n_pkts_hit = stats->n_pkts_hit[hit];
2142         n_pkts_action = stats->n_pkts_action[action_id];
2143
2144         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2145               p->thread_id,
2146               table_id,
2147               hit ? "hit" : "miss",
2148               (uint32_t)action_id);
2149
2150         t->action_id = action_id;
2151         t->structs[0] = action_data;
2152         t->hit = hit;
2153         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2154         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2155
2156         /* Thread. */
2157         thread_ip_action_call(p, t, action_id);
2158 }
2159
2160 static inline void
2161 instr_table_af_exec(struct rte_swx_pipeline *p)
2162 {
2163         struct thread *t = &p->threads[p->thread_id];
2164         struct instruction *ip = t->ip;
2165         uint32_t table_id = ip->table.table_id;
2166         struct rte_swx_table_state *ts = &t->table_state[table_id];
2167         struct table_runtime *table = &t->tables[table_id];
2168         struct table_statistics *stats = &p->table_stats[table_id];
2169         uint64_t action_id, n_pkts_hit, n_pkts_action;
2170         uint8_t *action_data;
2171         action_func_t action_func;
2172         int done, hit;
2173
2174         /* Table. */
2175         done = table->func(ts->obj,
2176                            table->mailbox,
2177                            table->key,
2178                            &action_id,
2179                            &action_data,
2180                            &hit);
2181         if (!done) {
2182                 /* Thread. */
2183                 TRACE("[Thread %2u] table %u (not finalized)\n",
2184                       p->thread_id,
2185                       table_id);
2186
2187                 thread_yield(p);
2188                 return;
2189         }
2190
2191         action_id = hit ? action_id : ts->default_action_id;
2192         action_data = hit ? action_data : ts->default_action_data;
2193         action_func = p->action_funcs[action_id];
2194         n_pkts_hit = stats->n_pkts_hit[hit];
2195         n_pkts_action = stats->n_pkts_action[action_id];
2196
2197         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2198               p->thread_id,
2199               table_id,
2200               hit ? "hit" : "miss",
2201               (uint32_t)action_id);
2202
2203         t->action_id = action_id;
2204         t->structs[0] = action_data;
2205         t->hit = hit;
2206         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2207         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2208
2209         /* Thread. */
2210         thread_ip_inc(p);
2211
2212         /* Action. */
2213         action_func(p);
2214 }
2215
2216 static inline void
2217 instr_selector_exec(struct rte_swx_pipeline *p)
2218 {
2219         struct thread *t = &p->threads[p->thread_id];
2220         struct instruction *ip = t->ip;
2221         uint32_t selector_id = ip->table.table_id;
2222         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2223         struct selector_runtime *selector = &t->selectors[selector_id];
2224         struct selector_statistics *stats = &p->selector_stats[selector_id];
2225         uint64_t n_pkts = stats->n_pkts;
2226         int done;
2227
2228         /* Table. */
2229         done = rte_swx_table_selector_select(ts->obj,
2230                            selector->mailbox,
2231                            selector->group_id_buffer,
2232                            selector->selector_buffer,
2233                            selector->member_id_buffer);
2234         if (!done) {
2235                 /* Thread. */
2236                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2237                       p->thread_id,
2238                       selector_id);
2239
2240                 thread_yield(p);
2241                 return;
2242         }
2243
2244
2245         TRACE("[Thread %2u] selector %u\n",
2246               p->thread_id,
2247               selector_id);
2248
2249         stats->n_pkts = n_pkts + 1;
2250
2251         /* Thread. */
2252         thread_ip_inc(p);
2253 }
2254
2255 static inline void
2256 instr_learner_exec(struct rte_swx_pipeline *p)
2257 {
2258         struct thread *t = &p->threads[p->thread_id];
2259         struct instruction *ip = t->ip;
2260         uint32_t learner_id = ip->table.table_id;
2261         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2262                 p->n_selectors + learner_id];
2263         struct learner_runtime *l = &t->learners[learner_id];
2264         struct learner_statistics *stats = &p->learner_stats[learner_id];
2265         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2266         uint8_t *action_data;
2267         int done, hit;
2268
2269         /* Table. */
2270         time = rte_get_tsc_cycles();
2271
2272         done = rte_swx_table_learner_lookup(ts->obj,
2273                                             l->mailbox,
2274                                             time,
2275                                             l->key,
2276                                             &action_id,
2277                                             &action_data,
2278                                             &hit);
2279         if (!done) {
2280                 /* Thread. */
2281                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2282                       p->thread_id,
2283                       learner_id);
2284
2285                 thread_yield(p);
2286                 return;
2287         }
2288
2289         action_id = hit ? action_id : ts->default_action_id;
2290         action_data = hit ? action_data : ts->default_action_data;
2291         n_pkts_hit = stats->n_pkts_hit[hit];
2292         n_pkts_action = stats->n_pkts_action[action_id];
2293
2294         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2295               p->thread_id,
2296               learner_id,
2297               hit ? "hit" : "miss",
2298               (uint32_t)action_id);
2299
2300         t->action_id = action_id;
2301         t->structs[0] = action_data;
2302         t->hit = hit;
2303         t->learner_id = learner_id;
2304         t->time = time;
2305         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2306         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2307
2308         /* Thread. */
2309         thread_ip_action_call(p, t, action_id);
2310 }
2311
2312 static inline void
2313 instr_learner_af_exec(struct rte_swx_pipeline *p)
2314 {
2315         struct thread *t = &p->threads[p->thread_id];
2316         struct instruction *ip = t->ip;
2317         uint32_t learner_id = ip->table.table_id;
2318         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2319                 p->n_selectors + learner_id];
2320         struct learner_runtime *l = &t->learners[learner_id];
2321         struct learner_statistics *stats = &p->learner_stats[learner_id];
2322         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2323         uint8_t *action_data;
2324         action_func_t action_func;
2325         int done, hit;
2326
2327         /* Table. */
2328         time = rte_get_tsc_cycles();
2329
2330         done = rte_swx_table_learner_lookup(ts->obj,
2331                                             l->mailbox,
2332                                             time,
2333                                             l->key,
2334                                             &action_id,
2335                                             &action_data,
2336                                             &hit);
2337         if (!done) {
2338                 /* Thread. */
2339                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2340                       p->thread_id,
2341                       learner_id);
2342
2343                 thread_yield(p);
2344                 return;
2345         }
2346
2347         action_id = hit ? action_id : ts->default_action_id;
2348         action_data = hit ? action_data : ts->default_action_data;
2349         action_func = p->action_funcs[action_id];
2350         n_pkts_hit = stats->n_pkts_hit[hit];
2351         n_pkts_action = stats->n_pkts_action[action_id];
2352
2353         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2354               p->thread_id,
2355               learner_id,
2356               hit ? "hit" : "miss",
2357               (uint32_t)action_id);
2358
2359         t->action_id = action_id;
2360         t->structs[0] = action_data;
2361         t->hit = hit;
2362         t->learner_id = learner_id;
2363         t->time = time;
2364         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2365         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2366
2367         /* Thread. */
2368         thread_ip_action_call(p, t, action_id);
2369
2370         /* Action */
2371         action_func(p);
2372 }
2373
2374 /*
2375  * learn.
2376  */
2377 static struct action *
2378 action_find(struct rte_swx_pipeline *p, const char *name);
2379
2380 static int
2381 action_has_nbo_args(struct action *a);
2382
2383 static int
2384 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2385
2386 static int
2387 instr_learn_translate(struct rte_swx_pipeline *p,
2388                       struct action *action,
2389                       char **tokens,
2390                       int n_tokens,
2391                       struct instruction *instr,
2392                       struct instruction_data *data __rte_unused)
2393 {
2394         struct action *a;
2395         const char *mf_name;
2396         uint32_t mf_offset = 0;
2397
2398         CHECK(action, EINVAL);
2399         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2400
2401         a = action_find(p, tokens[1]);
2402         CHECK(a, EINVAL);
2403         CHECK(!action_has_nbo_args(a), EINVAL);
2404
2405         mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2406         CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2407
2408         if (mf_name) {
2409                 struct field *mf;
2410
2411                 mf = metadata_field_parse(p, mf_name);
2412                 CHECK(mf, EINVAL);
2413
2414                 mf_offset = mf->offset / 8;
2415         }
2416
2417         instr->type = INSTR_LEARNER_LEARN;
2418         instr->learn.action_id = a->id;
2419         instr->learn.mf_offset = mf_offset;
2420
2421         return 0;
2422 }
2423
2424 static inline void
2425 instr_learn_exec(struct rte_swx_pipeline *p)
2426 {
2427         struct thread *t = &p->threads[p->thread_id];
2428         struct instruction *ip = t->ip;
2429
2430         __instr_learn_exec(p, t, ip);
2431
2432         /* Thread. */
2433         thread_ip_inc(p);
2434 }
2435
2436 /*
2437  * forget.
2438  */
2439 static int
2440 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2441                        struct action *action,
2442                        char **tokens __rte_unused,
2443                        int n_tokens,
2444                        struct instruction *instr,
2445                        struct instruction_data *data __rte_unused)
2446 {
2447         CHECK(action, EINVAL);
2448         CHECK(n_tokens == 1, EINVAL);
2449
2450         instr->type = INSTR_LEARNER_FORGET;
2451
2452         return 0;
2453 }
2454
2455 static inline void
2456 instr_forget_exec(struct rte_swx_pipeline *p)
2457 {
2458         struct thread *t = &p->threads[p->thread_id];
2459         struct instruction *ip = t->ip;
2460
2461         __instr_forget_exec(p, t, ip);
2462
2463         /* Thread. */
2464         thread_ip_inc(p);
2465 }
2466
2467 /*
2468  * extern.
2469  */
2470 static int
2471 instr_extern_translate(struct rte_swx_pipeline *p,
2472                        struct action *action __rte_unused,
2473                        char **tokens,
2474                        int n_tokens,
2475                        struct instruction *instr,
2476                        struct instruction_data *data __rte_unused)
2477 {
2478         char *token = tokens[1];
2479
2480         CHECK(n_tokens == 2, EINVAL);
2481
2482         if (token[0] == 'e') {
2483                 struct extern_obj *obj;
2484                 struct extern_type_member_func *func;
2485
2486                 func = extern_obj_member_func_parse(p, token, &obj);
2487                 CHECK(func, EINVAL);
2488
2489                 instr->type = INSTR_EXTERN_OBJ;
2490                 instr->ext_obj.ext_obj_id = obj->id;
2491                 instr->ext_obj.func_id = func->id;
2492
2493                 return 0;
2494         }
2495
2496         if (token[0] == 'f') {
2497                 struct extern_func *func;
2498
2499                 func = extern_func_parse(p, token);
2500                 CHECK(func, EINVAL);
2501
2502                 instr->type = INSTR_EXTERN_FUNC;
2503                 instr->ext_func.ext_func_id = func->id;
2504
2505                 return 0;
2506         }
2507
2508         CHECK(0, EINVAL);
2509 }
2510
2511 static inline void
2512 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2513 {
2514         struct thread *t = &p->threads[p->thread_id];
2515         struct instruction *ip = t->ip;
2516         uint32_t done;
2517
2518         /* Extern object member function execute. */
2519         done = __instr_extern_obj_exec(p, t, ip);
2520
2521         /* Thread. */
2522         thread_ip_inc_cond(t, done);
2523         thread_yield_cond(p, done ^ 1);
2524 }
2525
2526 static inline void
2527 instr_extern_func_exec(struct rte_swx_pipeline *p)
2528 {
2529         struct thread *t = &p->threads[p->thread_id];
2530         struct instruction *ip = t->ip;
2531         uint32_t done;
2532
2533         /* Extern function execute. */
2534         done = __instr_extern_func_exec(p, t, ip);
2535
2536         /* Thread. */
2537         thread_ip_inc_cond(t, done);
2538         thread_yield_cond(p, done ^ 1);
2539 }
2540
2541 /*
2542  * mov.
2543  */
2544 static int
2545 instr_mov_translate(struct rte_swx_pipeline *p,
2546                     struct action *action,
2547                     char **tokens,
2548                     int n_tokens,
2549                     struct instruction *instr,
2550                     struct instruction_data *data __rte_unused)
2551 {
2552         char *dst = tokens[1], *src = tokens[2];
2553         struct field *fdst, *fsrc;
2554         uint64_t src_val;
2555         uint32_t dst_struct_id = 0, src_struct_id = 0;
2556
2557         CHECK(n_tokens == 3, EINVAL);
2558
2559         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2560         CHECK(fdst, EINVAL);
2561         CHECK(!fdst->var_size, EINVAL);
2562
2563         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2564         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2565         if (fsrc) {
2566                 CHECK(!fsrc->var_size, EINVAL);
2567
2568                 instr->type = INSTR_MOV;
2569                 if (dst[0] != 'h' && src[0] == 'h')
2570                         instr->type = INSTR_MOV_MH;
2571                 if (dst[0] == 'h' && src[0] != 'h')
2572                         instr->type = INSTR_MOV_HM;
2573                 if (dst[0] == 'h' && src[0] == 'h')
2574                         instr->type = INSTR_MOV_HH;
2575
2576                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2577                 instr->mov.dst.n_bits = fdst->n_bits;
2578                 instr->mov.dst.offset = fdst->offset / 8;
2579                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2580                 instr->mov.src.n_bits = fsrc->n_bits;
2581                 instr->mov.src.offset = fsrc->offset / 8;
2582                 return 0;
2583         }
2584
2585         /* MOV_I. */
2586         src_val = strtoull(src, &src, 0);
2587         CHECK(!src[0], EINVAL);
2588
2589         if (dst[0] == 'h')
2590                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2591
2592         instr->type = INSTR_MOV_I;
2593         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2594         instr->mov.dst.n_bits = fdst->n_bits;
2595         instr->mov.dst.offset = fdst->offset / 8;
2596         instr->mov.src_val = src_val;
2597         return 0;
2598 }
2599
2600 static inline void
2601 instr_mov_exec(struct rte_swx_pipeline *p)
2602 {
2603         struct thread *t = &p->threads[p->thread_id];
2604         struct instruction *ip = t->ip;
2605
2606         __instr_mov_exec(p, t, ip);
2607
2608         /* Thread. */
2609         thread_ip_inc(p);
2610 }
2611
2612 static inline void
2613 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2614 {
2615         struct thread *t = &p->threads[p->thread_id];
2616         struct instruction *ip = t->ip;
2617
2618         __instr_mov_mh_exec(p, t, ip);
2619
2620         /* Thread. */
2621         thread_ip_inc(p);
2622 }
2623
2624 static inline void
2625 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2626 {
2627         struct thread *t = &p->threads[p->thread_id];
2628         struct instruction *ip = t->ip;
2629
2630         __instr_mov_hm_exec(p, t, ip);
2631
2632         /* Thread. */
2633         thread_ip_inc(p);
2634 }
2635
2636 static inline void
2637 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2638 {
2639         struct thread *t = &p->threads[p->thread_id];
2640         struct instruction *ip = t->ip;
2641
2642         __instr_mov_hh_exec(p, t, ip);
2643
2644         /* Thread. */
2645         thread_ip_inc(p);
2646 }
2647
2648 static inline void
2649 instr_mov_i_exec(struct rte_swx_pipeline *p)
2650 {
2651         struct thread *t = &p->threads[p->thread_id];
2652         struct instruction *ip = t->ip;
2653
2654         __instr_mov_i_exec(p, t, ip);
2655
2656         /* Thread. */
2657         thread_ip_inc(p);
2658 }
2659
2660 /*
2661  * dma.
2662  */
2663 static inline void
2664 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2665 {
2666         struct thread *t = &p->threads[p->thread_id];
2667         struct instruction *ip = t->ip;
2668
2669         __instr_dma_ht_exec(p, t, ip);
2670
2671         /* Thread. */
2672         thread_ip_inc(p);
2673 }
2674
2675 static inline void
2676 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2677 {
2678         struct thread *t = &p->threads[p->thread_id];
2679         struct instruction *ip = t->ip;
2680
2681         __instr_dma_ht2_exec(p, t, ip);
2682
2683         /* Thread. */
2684         thread_ip_inc(p);
2685 }
2686
2687 static inline void
2688 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2689 {
2690         struct thread *t = &p->threads[p->thread_id];
2691         struct instruction *ip = t->ip;
2692
2693         __instr_dma_ht3_exec(p, t, ip);
2694
2695         /* Thread. */
2696         thread_ip_inc(p);
2697 }
2698
2699 static inline void
2700 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2701 {
2702         struct thread *t = &p->threads[p->thread_id];
2703         struct instruction *ip = t->ip;
2704
2705         __instr_dma_ht4_exec(p, t, ip);
2706
2707         /* Thread. */
2708         thread_ip_inc(p);
2709 }
2710
2711 static inline void
2712 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2713 {
2714         struct thread *t = &p->threads[p->thread_id];
2715         struct instruction *ip = t->ip;
2716
2717         __instr_dma_ht5_exec(p, t, ip);
2718
2719         /* Thread. */
2720         thread_ip_inc(p);
2721 }
2722
2723 static inline void
2724 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2725 {
2726         struct thread *t = &p->threads[p->thread_id];
2727         struct instruction *ip = t->ip;
2728
2729         __instr_dma_ht6_exec(p, t, ip);
2730
2731         /* Thread. */
2732         thread_ip_inc(p);
2733 }
2734
2735 static inline void
2736 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2737 {
2738         struct thread *t = &p->threads[p->thread_id];
2739         struct instruction *ip = t->ip;
2740
2741         __instr_dma_ht7_exec(p, t, ip);
2742
2743         /* Thread. */
2744         thread_ip_inc(p);
2745 }
2746
2747 static inline void
2748 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2749 {
2750         struct thread *t = &p->threads[p->thread_id];
2751         struct instruction *ip = t->ip;
2752
2753         __instr_dma_ht8_exec(p, t, ip);
2754
2755         /* Thread. */
2756         thread_ip_inc(p);
2757 }
2758
2759 /*
2760  * alu.
2761  */
2762 static int
2763 instr_alu_add_translate(struct rte_swx_pipeline *p,
2764                         struct action *action,
2765                         char **tokens,
2766                         int n_tokens,
2767                         struct instruction *instr,
2768                         struct instruction_data *data __rte_unused)
2769 {
2770         char *dst = tokens[1], *src = tokens[2];
2771         struct field *fdst, *fsrc;
2772         uint64_t src_val;
2773         uint32_t dst_struct_id = 0, src_struct_id = 0;
2774
2775         CHECK(n_tokens == 3, EINVAL);
2776
2777         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2778         CHECK(fdst, EINVAL);
2779         CHECK(!fdst->var_size, EINVAL);
2780
2781         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2782         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2783         if (fsrc) {
2784                 CHECK(!fsrc->var_size, EINVAL);
2785
2786                 instr->type = INSTR_ALU_ADD;
2787                 if (dst[0] == 'h' && src[0] != 'h')
2788                         instr->type = INSTR_ALU_ADD_HM;
2789                 if (dst[0] != 'h' && src[0] == 'h')
2790                         instr->type = INSTR_ALU_ADD_MH;
2791                 if (dst[0] == 'h' && src[0] == 'h')
2792                         instr->type = INSTR_ALU_ADD_HH;
2793
2794                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2795                 instr->alu.dst.n_bits = fdst->n_bits;
2796                 instr->alu.dst.offset = fdst->offset / 8;
2797                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2798                 instr->alu.src.n_bits = fsrc->n_bits;
2799                 instr->alu.src.offset = fsrc->offset / 8;
2800                 return 0;
2801         }
2802
2803         /* ADD_MI, ADD_HI. */
2804         src_val = strtoull(src, &src, 0);
2805         CHECK(!src[0], EINVAL);
2806
2807         instr->type = INSTR_ALU_ADD_MI;
2808         if (dst[0] == 'h')
2809                 instr->type = INSTR_ALU_ADD_HI;
2810
2811         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2812         instr->alu.dst.n_bits = fdst->n_bits;
2813         instr->alu.dst.offset = fdst->offset / 8;
2814         instr->alu.src_val = src_val;
2815         return 0;
2816 }
2817
2818 static int
2819 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2820                         struct action *action,
2821                         char **tokens,
2822                         int n_tokens,
2823                         struct instruction *instr,
2824                         struct instruction_data *data __rte_unused)
2825 {
2826         char *dst = tokens[1], *src = tokens[2];
2827         struct field *fdst, *fsrc;
2828         uint64_t src_val;
2829         uint32_t dst_struct_id = 0, src_struct_id = 0;
2830
2831         CHECK(n_tokens == 3, EINVAL);
2832
2833         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2834         CHECK(fdst, EINVAL);
2835         CHECK(!fdst->var_size, EINVAL);
2836
2837         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2838         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2839         if (fsrc) {
2840                 CHECK(!fsrc->var_size, EINVAL);
2841
2842                 instr->type = INSTR_ALU_SUB;
2843                 if (dst[0] == 'h' && src[0] != 'h')
2844                         instr->type = INSTR_ALU_SUB_HM;
2845                 if (dst[0] != 'h' && src[0] == 'h')
2846                         instr->type = INSTR_ALU_SUB_MH;
2847                 if (dst[0] == 'h' && src[0] == 'h')
2848                         instr->type = INSTR_ALU_SUB_HH;
2849
2850                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2851                 instr->alu.dst.n_bits = fdst->n_bits;
2852                 instr->alu.dst.offset = fdst->offset / 8;
2853                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2854                 instr->alu.src.n_bits = fsrc->n_bits;
2855                 instr->alu.src.offset = fsrc->offset / 8;
2856                 return 0;
2857         }
2858
2859         /* SUB_MI, SUB_HI. */
2860         src_val = strtoull(src, &src, 0);
2861         CHECK(!src[0], EINVAL);
2862
2863         instr->type = INSTR_ALU_SUB_MI;
2864         if (dst[0] == 'h')
2865                 instr->type = INSTR_ALU_SUB_HI;
2866
2867         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2868         instr->alu.dst.n_bits = fdst->n_bits;
2869         instr->alu.dst.offset = fdst->offset / 8;
2870         instr->alu.src_val = src_val;
2871         return 0;
2872 }
2873
2874 static int
2875 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2876                           struct action *action __rte_unused,
2877                           char **tokens,
2878                           int n_tokens,
2879                           struct instruction *instr,
2880                           struct instruction_data *data __rte_unused)
2881 {
2882         char *dst = tokens[1], *src = tokens[2];
2883         struct header *hdst, *hsrc;
2884         struct field *fdst, *fsrc;
2885
2886         CHECK(n_tokens == 3, EINVAL);
2887
2888         fdst = header_field_parse(p, dst, &hdst);
2889         CHECK(fdst, EINVAL);
2890         CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
2891
2892         /* CKADD_FIELD. */
2893         fsrc = header_field_parse(p, src, &hsrc);
2894         if (fsrc) {
2895                 CHECK(!fsrc->var_size, EINVAL);
2896
2897                 instr->type = INSTR_ALU_CKADD_FIELD;
2898                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2899                 instr->alu.dst.n_bits = fdst->n_bits;
2900                 instr->alu.dst.offset = fdst->offset / 8;
2901                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2902                 instr->alu.src.n_bits = fsrc->n_bits;
2903                 instr->alu.src.offset = fsrc->offset / 8;
2904                 return 0;
2905         }
2906
2907         /* CKADD_STRUCT, CKADD_STRUCT20. */
2908         hsrc = header_parse(p, src);
2909         CHECK(hsrc, EINVAL);
2910
2911         instr->type = INSTR_ALU_CKADD_STRUCT;
2912         if (!hsrc->st->var_size && ((hsrc->st->n_bits / 8) == 20))
2913                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2914
2915         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2916         instr->alu.dst.n_bits = fdst->n_bits;
2917         instr->alu.dst.offset = fdst->offset / 8;
2918         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2919         instr->alu.src.n_bits = (uint8_t)hsrc->id; /* The src header ID is stored here. */
2920         instr->alu.src.offset = 0; /* Unused. */
2921         return 0;
2922 }
2923
2924 static int
2925 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2926                           struct action *action __rte_unused,
2927                           char **tokens,
2928                           int n_tokens,
2929                           struct instruction *instr,
2930                           struct instruction_data *data __rte_unused)
2931 {
2932         char *dst = tokens[1], *src = tokens[2];
2933         struct header *hdst, *hsrc;
2934         struct field *fdst, *fsrc;
2935
2936         CHECK(n_tokens == 3, EINVAL);
2937
2938         fdst = header_field_parse(p, dst, &hdst);
2939         CHECK(fdst, EINVAL);
2940         CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
2941
2942         fsrc = header_field_parse(p, src, &hsrc);
2943         CHECK(fsrc, EINVAL);
2944         CHECK(!fsrc->var_size, EINVAL);
2945
2946         instr->type = INSTR_ALU_CKSUB_FIELD;
2947         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2948         instr->alu.dst.n_bits = fdst->n_bits;
2949         instr->alu.dst.offset = fdst->offset / 8;
2950         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2951         instr->alu.src.n_bits = fsrc->n_bits;
2952         instr->alu.src.offset = fsrc->offset / 8;
2953         return 0;
2954 }
2955
2956 static int
2957 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2958                         struct action *action,
2959                         char **tokens,
2960                         int n_tokens,
2961                         struct instruction *instr,
2962                         struct instruction_data *data __rte_unused)
2963 {
2964         char *dst = tokens[1], *src = tokens[2];
2965         struct field *fdst, *fsrc;
2966         uint64_t src_val;
2967         uint32_t dst_struct_id = 0, src_struct_id = 0;
2968
2969         CHECK(n_tokens == 3, EINVAL);
2970
2971         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2972         CHECK(fdst, EINVAL);
2973         CHECK(!fdst->var_size, EINVAL);
2974
2975         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2976         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2977         if (fsrc) {
2978                 CHECK(!fsrc->var_size, EINVAL);
2979
2980                 instr->type = INSTR_ALU_SHL;
2981                 if (dst[0] == 'h' && src[0] != 'h')
2982                         instr->type = INSTR_ALU_SHL_HM;
2983                 if (dst[0] != 'h' && src[0] == 'h')
2984                         instr->type = INSTR_ALU_SHL_MH;
2985                 if (dst[0] == 'h' && src[0] == 'h')
2986                         instr->type = INSTR_ALU_SHL_HH;
2987
2988                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2989                 instr->alu.dst.n_bits = fdst->n_bits;
2990                 instr->alu.dst.offset = fdst->offset / 8;
2991                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2992                 instr->alu.src.n_bits = fsrc->n_bits;
2993                 instr->alu.src.offset = fsrc->offset / 8;
2994                 return 0;
2995         }
2996
2997         /* SHL_MI, SHL_HI. */
2998         src_val = strtoull(src, &src, 0);
2999         CHECK(!src[0], EINVAL);
3000
3001         instr->type = INSTR_ALU_SHL_MI;
3002         if (dst[0] == 'h')
3003                 instr->type = INSTR_ALU_SHL_HI;
3004
3005         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3006         instr->alu.dst.n_bits = fdst->n_bits;
3007         instr->alu.dst.offset = fdst->offset / 8;
3008         instr->alu.src_val = src_val;
3009         return 0;
3010 }
3011
3012 static int
3013 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3014                         struct action *action,
3015                         char **tokens,
3016                         int n_tokens,
3017                         struct instruction *instr,
3018                         struct instruction_data *data __rte_unused)
3019 {
3020         char *dst = tokens[1], *src = tokens[2];
3021         struct field *fdst, *fsrc;
3022         uint64_t src_val;
3023         uint32_t dst_struct_id = 0, src_struct_id = 0;
3024
3025         CHECK(n_tokens == 3, EINVAL);
3026
3027         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3028         CHECK(fdst, EINVAL);
3029         CHECK(!fdst->var_size, EINVAL);
3030
3031         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3032         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3033         if (fsrc) {
3034                 CHECK(!fsrc->var_size, EINVAL);
3035
3036                 instr->type = INSTR_ALU_SHR;
3037                 if (dst[0] == 'h' && src[0] != 'h')
3038                         instr->type = INSTR_ALU_SHR_HM;
3039                 if (dst[0] != 'h' && src[0] == 'h')
3040                         instr->type = INSTR_ALU_SHR_MH;
3041                 if (dst[0] == 'h' && src[0] == 'h')
3042                         instr->type = INSTR_ALU_SHR_HH;
3043
3044                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3045                 instr->alu.dst.n_bits = fdst->n_bits;
3046                 instr->alu.dst.offset = fdst->offset / 8;
3047                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3048                 instr->alu.src.n_bits = fsrc->n_bits;
3049                 instr->alu.src.offset = fsrc->offset / 8;
3050                 return 0;
3051         }
3052
3053         /* SHR_MI, SHR_HI. */
3054         src_val = strtoull(src, &src, 0);
3055         CHECK(!src[0], EINVAL);
3056
3057         instr->type = INSTR_ALU_SHR_MI;
3058         if (dst[0] == 'h')
3059                 instr->type = INSTR_ALU_SHR_HI;
3060
3061         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3062         instr->alu.dst.n_bits = fdst->n_bits;
3063         instr->alu.dst.offset = fdst->offset / 8;
3064         instr->alu.src_val = src_val;
3065         return 0;
3066 }
3067
3068 static int
3069 instr_alu_and_translate(struct rte_swx_pipeline *p,
3070                         struct action *action,
3071                         char **tokens,
3072                         int n_tokens,
3073                         struct instruction *instr,
3074                         struct instruction_data *data __rte_unused)
3075 {
3076         char *dst = tokens[1], *src = tokens[2];
3077         struct field *fdst, *fsrc;
3078         uint64_t src_val;
3079         uint32_t dst_struct_id = 0, src_struct_id = 0;
3080
3081         CHECK(n_tokens == 3, EINVAL);
3082
3083         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3084         CHECK(fdst, EINVAL);
3085         CHECK(!fdst->var_size, EINVAL);
3086
3087         /* AND, AND_MH, AND_HM, AND_HH. */
3088         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3089         if (fsrc) {
3090                 CHECK(!fsrc->var_size, EINVAL);
3091
3092                 instr->type = INSTR_ALU_AND;
3093                 if (dst[0] != 'h' && src[0] == 'h')
3094                         instr->type = INSTR_ALU_AND_MH;
3095                 if (dst[0] == 'h' && src[0] != 'h')
3096                         instr->type = INSTR_ALU_AND_HM;
3097                 if (dst[0] == 'h' && src[0] == 'h')
3098                         instr->type = INSTR_ALU_AND_HH;
3099
3100                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3101                 instr->alu.dst.n_bits = fdst->n_bits;
3102                 instr->alu.dst.offset = fdst->offset / 8;
3103                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3104                 instr->alu.src.n_bits = fsrc->n_bits;
3105                 instr->alu.src.offset = fsrc->offset / 8;
3106                 return 0;
3107         }
3108
3109         /* AND_I. */
3110         src_val = strtoull(src, &src, 0);
3111         CHECK(!src[0], EINVAL);
3112
3113         if (dst[0] == 'h')
3114                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3115
3116         instr->type = INSTR_ALU_AND_I;
3117         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3118         instr->alu.dst.n_bits = fdst->n_bits;
3119         instr->alu.dst.offset = fdst->offset / 8;
3120         instr->alu.src_val = src_val;
3121         return 0;
3122 }
3123
3124 static int
3125 instr_alu_or_translate(struct rte_swx_pipeline *p,
3126                        struct action *action,
3127                        char **tokens,
3128                        int n_tokens,
3129                        struct instruction *instr,
3130                        struct instruction_data *data __rte_unused)
3131 {
3132         char *dst = tokens[1], *src = tokens[2];
3133         struct field *fdst, *fsrc;
3134         uint64_t src_val;
3135         uint32_t dst_struct_id = 0, src_struct_id = 0;
3136
3137         CHECK(n_tokens == 3, EINVAL);
3138
3139         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3140         CHECK(fdst, EINVAL);
3141         CHECK(!fdst->var_size, EINVAL);
3142
3143         /* OR, OR_MH, OR_HM, OR_HH. */
3144         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3145         if (fsrc) {
3146                 CHECK(!fsrc->var_size, EINVAL);
3147
3148                 instr->type = INSTR_ALU_OR;
3149                 if (dst[0] != 'h' && src[0] == 'h')
3150                         instr->type = INSTR_ALU_OR_MH;
3151                 if (dst[0] == 'h' && src[0] != 'h')
3152                         instr->type = INSTR_ALU_OR_HM;
3153                 if (dst[0] == 'h' && src[0] == 'h')
3154                         instr->type = INSTR_ALU_OR_HH;
3155
3156                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3157                 instr->alu.dst.n_bits = fdst->n_bits;
3158                 instr->alu.dst.offset = fdst->offset / 8;
3159                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3160                 instr->alu.src.n_bits = fsrc->n_bits;
3161                 instr->alu.src.offset = fsrc->offset / 8;
3162                 return 0;
3163         }
3164
3165         /* OR_I. */
3166         src_val = strtoull(src, &src, 0);
3167         CHECK(!src[0], EINVAL);
3168
3169         if (dst[0] == 'h')
3170                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3171
3172         instr->type = INSTR_ALU_OR_I;
3173         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3174         instr->alu.dst.n_bits = fdst->n_bits;
3175         instr->alu.dst.offset = fdst->offset / 8;
3176         instr->alu.src_val = src_val;
3177         return 0;
3178 }
3179
3180 static int
3181 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3182                         struct action *action,
3183                         char **tokens,
3184                         int n_tokens,
3185                         struct instruction *instr,
3186                         struct instruction_data *data __rte_unused)
3187 {
3188         char *dst = tokens[1], *src = tokens[2];
3189         struct field *fdst, *fsrc;
3190         uint64_t src_val;
3191         uint32_t dst_struct_id = 0, src_struct_id = 0;
3192
3193         CHECK(n_tokens == 3, EINVAL);
3194
3195         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3196         CHECK(fdst, EINVAL);
3197         CHECK(!fdst->var_size, EINVAL);
3198
3199         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3200         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3201         if (fsrc) {
3202                 CHECK(!fsrc->var_size, EINVAL);
3203
3204                 instr->type = INSTR_ALU_XOR;
3205                 if (dst[0] != 'h' && src[0] == 'h')
3206                         instr->type = INSTR_ALU_XOR_MH;
3207                 if (dst[0] == 'h' && src[0] != 'h')
3208                         instr->type = INSTR_ALU_XOR_HM;
3209                 if (dst[0] == 'h' && src[0] == 'h')
3210                         instr->type = INSTR_ALU_XOR_HH;
3211
3212                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3213                 instr->alu.dst.n_bits = fdst->n_bits;
3214                 instr->alu.dst.offset = fdst->offset / 8;
3215                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3216                 instr->alu.src.n_bits = fsrc->n_bits;
3217                 instr->alu.src.offset = fsrc->offset / 8;
3218                 return 0;
3219         }
3220
3221         /* XOR_I. */
3222         src_val = strtoull(src, &src, 0);
3223         CHECK(!src[0], EINVAL);
3224
3225         if (dst[0] == 'h')
3226                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3227
3228         instr->type = INSTR_ALU_XOR_I;
3229         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3230         instr->alu.dst.n_bits = fdst->n_bits;
3231         instr->alu.dst.offset = fdst->offset / 8;
3232         instr->alu.src_val = src_val;
3233         return 0;
3234 }
3235
3236 static inline void
3237 instr_alu_add_exec(struct rte_swx_pipeline *p)
3238 {
3239         struct thread *t = &p->threads[p->thread_id];
3240         struct instruction *ip = t->ip;
3241
3242         /* Structs */
3243         __instr_alu_add_exec(p, t, ip);
3244
3245         /* Thread. */
3246         thread_ip_inc(p);
3247 }
3248
3249 static inline void
3250 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3251 {
3252         struct thread *t = &p->threads[p->thread_id];
3253         struct instruction *ip = t->ip;
3254
3255         /* Structs. */
3256         __instr_alu_add_mh_exec(p, t, ip);
3257
3258         /* Thread. */
3259         thread_ip_inc(p);
3260 }
3261
3262 static inline void
3263 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3264 {
3265         struct thread *t = &p->threads[p->thread_id];
3266         struct instruction *ip = t->ip;
3267
3268         /* Structs. */
3269         __instr_alu_add_hm_exec(p, t, ip);
3270
3271         /* Thread. */
3272         thread_ip_inc(p);
3273 }
3274
3275 static inline void
3276 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3277 {
3278         struct thread *t = &p->threads[p->thread_id];
3279         struct instruction *ip = t->ip;
3280
3281         /* Structs. */
3282         __instr_alu_add_hh_exec(p, t, ip);
3283
3284         /* Thread. */
3285         thread_ip_inc(p);
3286 }
3287
3288 static inline void
3289 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3290 {
3291         struct thread *t = &p->threads[p->thread_id];
3292         struct instruction *ip = t->ip;
3293
3294         /* Structs. */
3295         __instr_alu_add_mi_exec(p, t, ip);
3296
3297         /* Thread. */
3298         thread_ip_inc(p);
3299 }
3300
3301 static inline void
3302 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3303 {
3304         struct thread *t = &p->threads[p->thread_id];
3305         struct instruction *ip = t->ip;
3306
3307         /* Structs. */
3308         __instr_alu_add_hi_exec(p, t, ip);
3309
3310         /* Thread. */
3311         thread_ip_inc(p);
3312 }
3313
3314 static inline void
3315 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3316 {
3317         struct thread *t = &p->threads[p->thread_id];
3318         struct instruction *ip = t->ip;
3319
3320         /* Structs. */
3321         __instr_alu_sub_exec(p, t, ip);
3322
3323         /* Thread. */
3324         thread_ip_inc(p);
3325 }
3326
3327 static inline void
3328 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3329 {
3330         struct thread *t = &p->threads[p->thread_id];
3331         struct instruction *ip = t->ip;
3332
3333         /* Structs. */
3334         __instr_alu_sub_mh_exec(p, t, ip);
3335
3336         /* Thread. */
3337         thread_ip_inc(p);
3338 }
3339
3340 static inline void
3341 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3342 {
3343         struct thread *t = &p->threads[p->thread_id];
3344         struct instruction *ip = t->ip;
3345
3346         /* Structs. */
3347         __instr_alu_sub_hm_exec(p, t, ip);
3348
3349         /* Thread. */
3350         thread_ip_inc(p);
3351 }
3352
3353 static inline void
3354 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3355 {
3356         struct thread *t = &p->threads[p->thread_id];
3357         struct instruction *ip = t->ip;
3358
3359         /* Structs. */
3360         __instr_alu_sub_hh_exec(p, t, ip);
3361
3362         /* Thread. */
3363         thread_ip_inc(p);
3364 }
3365
3366 static inline void
3367 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3368 {
3369         struct thread *t = &p->threads[p->thread_id];
3370         struct instruction *ip = t->ip;
3371
3372         /* Structs. */
3373         __instr_alu_sub_mi_exec(p, t, ip);
3374
3375         /* Thread. */
3376         thread_ip_inc(p);
3377 }
3378
3379 static inline void
3380 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3381 {
3382         struct thread *t = &p->threads[p->thread_id];
3383         struct instruction *ip = t->ip;
3384
3385         /* Structs. */
3386         __instr_alu_sub_hi_exec(p, t, ip);
3387
3388         /* Thread. */
3389         thread_ip_inc(p);
3390 }
3391
3392 static inline void
3393 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3394 {
3395         struct thread *t = &p->threads[p->thread_id];
3396         struct instruction *ip = t->ip;
3397
3398         /* Structs. */
3399         __instr_alu_shl_exec(p, t, ip);
3400
3401         /* Thread. */
3402         thread_ip_inc(p);
3403 }
3404
3405 static inline void
3406 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3407 {
3408         struct thread *t = &p->threads[p->thread_id];
3409         struct instruction *ip = t->ip;
3410
3411         /* Structs. */
3412         __instr_alu_shl_mh_exec(p, t, ip);
3413
3414         /* Thread. */
3415         thread_ip_inc(p);
3416 }
3417
3418 static inline void
3419 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3420 {
3421         struct thread *t = &p->threads[p->thread_id];
3422         struct instruction *ip = t->ip;
3423
3424         /* Structs. */
3425         __instr_alu_shl_hm_exec(p, t, ip);
3426
3427         /* Thread. */
3428         thread_ip_inc(p);
3429 }
3430
3431 static inline void
3432 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3433 {
3434         struct thread *t = &p->threads[p->thread_id];
3435         struct instruction *ip = t->ip;
3436
3437         /* Structs. */
3438         __instr_alu_shl_hh_exec(p, t, ip);
3439
3440         /* Thread. */
3441         thread_ip_inc(p);
3442 }
3443
3444 static inline void
3445 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3446 {
3447         struct thread *t = &p->threads[p->thread_id];
3448         struct instruction *ip = t->ip;
3449
3450         /* Structs. */
3451         __instr_alu_shl_mi_exec(p, t, ip);
3452
3453         /* Thread. */
3454         thread_ip_inc(p);
3455 }
3456
3457 static inline void
3458 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3459 {
3460         struct thread *t = &p->threads[p->thread_id];
3461         struct instruction *ip = t->ip;
3462
3463         /* Structs. */
3464         __instr_alu_shl_hi_exec(p, t, ip);
3465
3466         /* Thread. */
3467         thread_ip_inc(p);
3468 }
3469
3470 static inline void
3471 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3472 {
3473         struct thread *t = &p->threads[p->thread_id];
3474         struct instruction *ip = t->ip;
3475
3476         /* Structs. */
3477         __instr_alu_shr_exec(p, t, ip);
3478
3479         /* Thread. */
3480         thread_ip_inc(p);
3481 }
3482
3483 static inline void
3484 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3485 {
3486         struct thread *t = &p->threads[p->thread_id];
3487         struct instruction *ip = t->ip;
3488
3489         /* Structs. */
3490         __instr_alu_shr_mh_exec(p, t, ip);
3491
3492         /* Thread. */
3493         thread_ip_inc(p);
3494 }
3495
3496 static inline void
3497 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3498 {
3499         struct thread *t = &p->threads[p->thread_id];
3500         struct instruction *ip = t->ip;
3501
3502         /* Structs. */
3503         __instr_alu_shr_hm_exec(p, t, ip);
3504
3505         /* Thread. */
3506         thread_ip_inc(p);
3507 }
3508
3509 static inline void
3510 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3511 {
3512         struct thread *t = &p->threads[p->thread_id];
3513         struct instruction *ip = t->ip;
3514
3515         /* Structs. */
3516         __instr_alu_shr_hh_exec(p, t, ip);
3517
3518         /* Thread. */
3519         thread_ip_inc(p);
3520 }
3521
3522 static inline void
3523 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3524 {
3525         struct thread *t = &p->threads[p->thread_id];
3526         struct instruction *ip = t->ip;
3527
3528         /* Structs. */
3529         __instr_alu_shr_mi_exec(p, t, ip);
3530
3531         /* Thread. */
3532         thread_ip_inc(p);
3533 }
3534
3535 static inline void
3536 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3537 {
3538         struct thread *t = &p->threads[p->thread_id];
3539         struct instruction *ip = t->ip;
3540
3541         /* Structs. */
3542         __instr_alu_shr_hi_exec(p, t, ip);
3543
3544         /* Thread. */
3545         thread_ip_inc(p);
3546 }
3547
3548 static inline void
3549 instr_alu_and_exec(struct rte_swx_pipeline *p)
3550 {
3551         struct thread *t = &p->threads[p->thread_id];
3552         struct instruction *ip = t->ip;
3553
3554         /* Structs. */
3555         __instr_alu_and_exec(p, t, ip);
3556
3557         /* Thread. */
3558         thread_ip_inc(p);
3559 }
3560
3561 static inline void
3562 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3563 {
3564         struct thread *t = &p->threads[p->thread_id];
3565         struct instruction *ip = t->ip;
3566
3567         /* Structs. */
3568         __instr_alu_and_mh_exec(p, t, ip);
3569
3570         /* Thread. */
3571         thread_ip_inc(p);
3572 }
3573
3574 static inline void
3575 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3576 {
3577         struct thread *t = &p->threads[p->thread_id];
3578         struct instruction *ip = t->ip;
3579
3580         /* Structs. */
3581         __instr_alu_and_hm_exec(p, t, ip);
3582
3583         /* Thread. */
3584         thread_ip_inc(p);
3585 }
3586
3587 static inline void
3588 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3589 {
3590         struct thread *t = &p->threads[p->thread_id];
3591         struct instruction *ip = t->ip;
3592
3593         /* Structs. */
3594         __instr_alu_and_hh_exec(p, t, ip);
3595
3596         /* Thread. */
3597         thread_ip_inc(p);
3598 }
3599
3600 static inline void
3601 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3602 {
3603         struct thread *t = &p->threads[p->thread_id];
3604         struct instruction *ip = t->ip;
3605
3606         /* Structs. */
3607         __instr_alu_and_i_exec(p, t, ip);
3608
3609         /* Thread. */
3610         thread_ip_inc(p);
3611 }
3612
3613 static inline void
3614 instr_alu_or_exec(struct rte_swx_pipeline *p)
3615 {
3616         struct thread *t = &p->threads[p->thread_id];
3617         struct instruction *ip = t->ip;
3618
3619         /* Structs. */
3620         __instr_alu_or_exec(p, t, ip);
3621
3622         /* Thread. */
3623         thread_ip_inc(p);
3624 }
3625
3626 static inline void
3627 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3628 {
3629         struct thread *t = &p->threads[p->thread_id];
3630         struct instruction *ip = t->ip;
3631
3632         /* Structs. */
3633         __instr_alu_or_mh_exec(p, t, ip);
3634
3635         /* Thread. */
3636         thread_ip_inc(p);
3637 }
3638
3639 static inline void
3640 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3641 {
3642         struct thread *t = &p->threads[p->thread_id];
3643         struct instruction *ip = t->ip;
3644
3645         /* Structs. */
3646         __instr_alu_or_hm_exec(p, t, ip);
3647
3648         /* Thread. */
3649         thread_ip_inc(p);
3650 }
3651
3652 static inline void
3653 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3654 {
3655         struct thread *t = &p->threads[p->thread_id];
3656         struct instruction *ip = t->ip;
3657
3658         /* Structs. */
3659         __instr_alu_or_hh_exec(p, t, ip);
3660
3661         /* Thread. */
3662         thread_ip_inc(p);
3663 }
3664
3665 static inline void
3666 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3667 {
3668         struct thread *t = &p->threads[p->thread_id];
3669         struct instruction *ip = t->ip;
3670
3671         /* Structs. */
3672         __instr_alu_or_i_exec(p, t, ip);
3673
3674         /* Thread. */
3675         thread_ip_inc(p);
3676 }
3677
3678 static inline void
3679 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3680 {
3681         struct thread *t = &p->threads[p->thread_id];
3682         struct instruction *ip = t->ip;
3683
3684         /* Structs. */
3685         __instr_alu_xor_exec(p, t, ip);
3686
3687         /* Thread. */
3688         thread_ip_inc(p);
3689 }
3690
3691 static inline void
3692 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3693 {
3694         struct thread *t = &p->threads[p->thread_id];
3695         struct instruction *ip = t->ip;
3696
3697         /* Structs. */
3698         __instr_alu_xor_mh_exec(p, t, ip);
3699
3700         /* Thread. */
3701         thread_ip_inc(p);
3702 }
3703
3704 static inline void
3705 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3706 {
3707         struct thread *t = &p->threads[p->thread_id];
3708         struct instruction *ip = t->ip;
3709
3710         /* Structs. */
3711         __instr_alu_xor_hm_exec(p, t, ip);
3712
3713         /* Thread. */
3714         thread_ip_inc(p);
3715 }
3716
3717 static inline void
3718 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3719 {
3720         struct thread *t = &p->threads[p->thread_id];
3721         struct instruction *ip = t->ip;
3722
3723         /* Structs. */
3724         __instr_alu_xor_hh_exec(p, t, ip);
3725
3726         /* Thread. */
3727         thread_ip_inc(p);
3728 }
3729
3730 static inline void
3731 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3732 {
3733         struct thread *t = &p->threads[p->thread_id];
3734         struct instruction *ip = t->ip;
3735
3736         /* Structs. */
3737         __instr_alu_xor_i_exec(p, t, ip);
3738
3739         /* Thread. */
3740         thread_ip_inc(p);
3741 }
3742
3743 static inline void
3744 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3745 {
3746         struct thread *t = &p->threads[p->thread_id];
3747         struct instruction *ip = t->ip;
3748
3749         /* Structs. */
3750         __instr_alu_ckadd_field_exec(p, t, ip);
3751
3752         /* Thread. */
3753         thread_ip_inc(p);
3754 }
3755
3756 static inline void
3757 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3758 {
3759         struct thread *t = &p->threads[p->thread_id];
3760         struct instruction *ip = t->ip;
3761
3762         /* Structs. */
3763         __instr_alu_cksub_field_exec(p, t, ip);
3764
3765         /* Thread. */
3766         thread_ip_inc(p);
3767 }
3768
3769 static inline void
3770 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3771 {
3772         struct thread *t = &p->threads[p->thread_id];
3773         struct instruction *ip = t->ip;
3774
3775         /* Structs. */
3776         __instr_alu_ckadd_struct20_exec(p, t, ip);
3777
3778         /* Thread. */
3779         thread_ip_inc(p);
3780 }
3781
3782 static inline void
3783 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3784 {
3785         struct thread *t = &p->threads[p->thread_id];
3786         struct instruction *ip = t->ip;
3787
3788         /* Structs. */
3789         __instr_alu_ckadd_struct_exec(p, t, ip);
3790
3791         /* Thread. */
3792         thread_ip_inc(p);
3793 }
3794
3795 /*
3796  * Register array.
3797  */
3798 static struct regarray *
3799 regarray_find(struct rte_swx_pipeline *p, const char *name);
3800
3801 static int
3802 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3803                       struct action *action,
3804                       char **tokens,
3805                       int n_tokens,
3806                       struct instruction *instr,
3807                       struct instruction_data *data __rte_unused)
3808 {
3809         char *regarray = tokens[1], *idx = tokens[2];
3810         struct regarray *r;
3811         struct field *fidx;
3812         uint32_t idx_struct_id, idx_val;
3813
3814         CHECK(n_tokens == 3, EINVAL);
3815
3816         r = regarray_find(p, regarray);
3817         CHECK(r, EINVAL);
3818
3819         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3820         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3821         if (fidx) {
3822                 CHECK(!fidx->var_size, EINVAL);
3823
3824                 instr->type = INSTR_REGPREFETCH_RM;
3825                 if (idx[0] == 'h')
3826                         instr->type = INSTR_REGPREFETCH_RH;
3827
3828                 instr->regarray.regarray_id = r->id;
3829                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3830                 instr->regarray.idx.n_bits = fidx->n_bits;
3831                 instr->regarray.idx.offset = fidx->offset / 8;
3832                 instr->regarray.dstsrc_val = 0; /* Unused. */
3833                 return 0;
3834         }
3835
3836         /* REGPREFETCH_RI. */
3837         idx_val = strtoul(idx, &idx, 0);
3838         CHECK(!idx[0], EINVAL);
3839
3840         instr->type = INSTR_REGPREFETCH_RI;
3841         instr->regarray.regarray_id = r->id;
3842         instr->regarray.idx_val = idx_val;
3843         instr->regarray.dstsrc_val = 0; /* Unused. */
3844         return 0;
3845 }
3846
3847 static int
3848 instr_regrd_translate(struct rte_swx_pipeline *p,
3849                       struct action *action,
3850                       char **tokens,
3851                       int n_tokens,
3852                       struct instruction *instr,
3853                       struct instruction_data *data __rte_unused)
3854 {
3855         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3856         struct regarray *r;
3857         struct field *fdst, *fidx;
3858         uint32_t dst_struct_id, idx_struct_id, idx_val;
3859
3860         CHECK(n_tokens == 4, EINVAL);
3861
3862         r = regarray_find(p, regarray);
3863         CHECK(r, EINVAL);
3864
3865         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3866         CHECK(fdst, EINVAL);
3867         CHECK(!fdst->var_size, EINVAL);
3868
3869         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3870         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3871         if (fidx) {
3872                 CHECK(!fidx->var_size, EINVAL);
3873
3874                 instr->type = INSTR_REGRD_MRM;
3875                 if (dst[0] == 'h' && idx[0] != 'h')
3876                         instr->type = INSTR_REGRD_HRM;
3877                 if (dst[0] != 'h' && idx[0] == 'h')
3878                         instr->type = INSTR_REGRD_MRH;
3879                 if (dst[0] == 'h' && idx[0] == 'h')
3880                         instr->type = INSTR_REGRD_HRH;
3881
3882                 instr->regarray.regarray_id = r->id;
3883                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3884                 instr->regarray.idx.n_bits = fidx->n_bits;
3885                 instr->regarray.idx.offset = fidx->offset / 8;
3886                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3887                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3888                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3889                 return 0;
3890         }
3891
3892         /* REGRD_MRI, REGRD_HRI. */
3893         idx_val = strtoul(idx, &idx, 0);
3894         CHECK(!idx[0], EINVAL);
3895
3896         instr->type = INSTR_REGRD_MRI;
3897         if (dst[0] == 'h')
3898                 instr->type = INSTR_REGRD_HRI;
3899
3900         instr->regarray.regarray_id = r->id;
3901         instr->regarray.idx_val = idx_val;
3902         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3903         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3904         instr->regarray.dstsrc.offset = fdst->offset / 8;
3905         return 0;
3906 }
3907
3908 static int
3909 instr_regwr_translate(struct rte_swx_pipeline *p,
3910                       struct action *action,
3911                       char **tokens,
3912                       int n_tokens,
3913                       struct instruction *instr,
3914                       struct instruction_data *data __rte_unused)
3915 {
3916         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3917         struct regarray *r;
3918         struct field *fidx, *fsrc;
3919         uint64_t src_val;
3920         uint32_t idx_struct_id, idx_val, src_struct_id;
3921
3922         CHECK(n_tokens == 4, EINVAL);
3923
3924         r = regarray_find(p, regarray);
3925         CHECK(r, EINVAL);
3926
3927         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3928         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3929         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3930         if (fidx && fsrc) {
3931                 CHECK(!fidx->var_size, EINVAL);
3932                 CHECK(!fsrc->var_size, EINVAL);
3933
3934                 instr->type = INSTR_REGWR_RMM;
3935                 if (idx[0] == 'h' && src[0] != 'h')
3936                         instr->type = INSTR_REGWR_RHM;
3937                 if (idx[0] != 'h' && src[0] == 'h')
3938                         instr->type = INSTR_REGWR_RMH;
3939                 if (idx[0] == 'h' && src[0] == 'h')
3940                         instr->type = INSTR_REGWR_RHH;
3941
3942                 instr->regarray.regarray_id = r->id;
3943                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3944                 instr->regarray.idx.n_bits = fidx->n_bits;
3945                 instr->regarray.idx.offset = fidx->offset / 8;
3946                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3947                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3948                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3949                 return 0;
3950         }
3951
3952         /* REGWR_RHI, REGWR_RMI. */
3953         if (fidx && !fsrc) {
3954                 CHECK(!fidx->var_size, EINVAL);
3955
3956                 src_val = strtoull(src, &src, 0);
3957                 CHECK(!src[0], EINVAL);
3958
3959                 instr->type = INSTR_REGWR_RMI;
3960                 if (idx[0] == 'h')
3961                         instr->type = INSTR_REGWR_RHI;
3962
3963                 instr->regarray.regarray_id = r->id;
3964                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3965                 instr->regarray.idx.n_bits = fidx->n_bits;
3966                 instr->regarray.idx.offset = fidx->offset / 8;
3967                 instr->regarray.dstsrc_val = src_val;
3968                 return 0;
3969         }
3970
3971         /* REGWR_RIH, REGWR_RIM. */
3972         if (!fidx && fsrc) {
3973                 idx_val = strtoul(idx, &idx, 0);
3974                 CHECK(!idx[0], EINVAL);
3975
3976                 CHECK(!fsrc->var_size, EINVAL);
3977
3978                 instr->type = INSTR_REGWR_RIM;
3979                 if (src[0] == 'h')
3980                         instr->type = INSTR_REGWR_RIH;
3981
3982                 instr->regarray.regarray_id = r->id;
3983                 instr->regarray.idx_val = idx_val;
3984                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3985                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3986                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3987                 return 0;
3988         }
3989
3990         /* REGWR_RII. */
3991         src_val = strtoull(src, &src, 0);
3992         CHECK(!src[0], EINVAL);
3993
3994         idx_val = strtoul(idx, &idx, 0);
3995         CHECK(!idx[0], EINVAL);
3996
3997         instr->type = INSTR_REGWR_RII;
3998         instr->regarray.idx_val = idx_val;
3999         instr->regarray.dstsrc_val = src_val;
4000
4001         return 0;
4002 }
4003
4004 static int
4005 instr_regadd_translate(struct rte_swx_pipeline *p,
4006                        struct action *action,
4007                        char **tokens,
4008                        int n_tokens,
4009                        struct instruction *instr,
4010                        struct instruction_data *data __rte_unused)
4011 {
4012         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4013         struct regarray *r;
4014         struct field *fidx, *fsrc;
4015         uint64_t src_val;
4016         uint32_t idx_struct_id, idx_val, src_struct_id;
4017
4018         CHECK(n_tokens == 4, EINVAL);
4019
4020         r = regarray_find(p, regarray);
4021         CHECK(r, EINVAL);
4022
4023         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4024         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4025         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4026         if (fidx && fsrc) {
4027                 CHECK(!fidx->var_size, EINVAL);
4028                 CHECK(!fsrc->var_size, EINVAL);
4029
4030                 instr->type = INSTR_REGADD_RMM;
4031                 if (idx[0] == 'h' && src[0] != 'h')
4032                         instr->type = INSTR_REGADD_RHM;
4033                 if (idx[0] != 'h' && src[0] == 'h')
4034                         instr->type = INSTR_REGADD_RMH;
4035                 if (idx[0] == 'h' && src[0] == 'h')
4036                         instr->type = INSTR_REGADD_RHH;
4037
4038                 instr->regarray.regarray_id = r->id;
4039                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4040                 instr->regarray.idx.n_bits = fidx->n_bits;
4041                 instr->regarray.idx.offset = fidx->offset / 8;
4042                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4043                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4044                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4045                 return 0;
4046         }
4047
4048         /* REGADD_RHI, REGADD_RMI. */
4049         if (fidx && !fsrc) {
4050                 CHECK(!fidx->var_size, EINVAL);
4051
4052                 src_val = strtoull(src, &src, 0);
4053                 CHECK(!src[0], EINVAL);
4054
4055                 instr->type = INSTR_REGADD_RMI;
4056                 if (idx[0] == 'h')
4057                         instr->type = INSTR_REGADD_RHI;
4058
4059                 instr->regarray.regarray_id = r->id;
4060                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4061                 instr->regarray.idx.n_bits = fidx->n_bits;
4062                 instr->regarray.idx.offset = fidx->offset / 8;
4063                 instr->regarray.dstsrc_val = src_val;
4064                 return 0;
4065         }
4066
4067         /* REGADD_RIH, REGADD_RIM. */
4068         if (!fidx && fsrc) {
4069                 idx_val = strtoul(idx, &idx, 0);
4070                 CHECK(!idx[0], EINVAL);
4071
4072                 CHECK(!fsrc->var_size, EINVAL);
4073
4074                 instr->type = INSTR_REGADD_RIM;
4075                 if (src[0] == 'h')
4076                         instr->type = INSTR_REGADD_RIH;
4077
4078                 instr->regarray.regarray_id = r->id;
4079                 instr->regarray.idx_val = idx_val;
4080                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4081                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4082                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4083                 return 0;
4084         }
4085
4086         /* REGADD_RII. */
4087         src_val = strtoull(src, &src, 0);
4088         CHECK(!src[0], EINVAL);
4089
4090         idx_val = strtoul(idx, &idx, 0);
4091         CHECK(!idx[0], EINVAL);
4092
4093         instr->type = INSTR_REGADD_RII;
4094         instr->regarray.idx_val = idx_val;
4095         instr->regarray.dstsrc_val = src_val;
4096         return 0;
4097 }
4098
4099 static inline void
4100 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4101 {
4102         struct thread *t = &p->threads[p->thread_id];
4103         struct instruction *ip = t->ip;
4104
4105         /* Structs. */
4106         __instr_regprefetch_rh_exec(p, t, ip);
4107
4108         /* Thread. */
4109         thread_ip_inc(p);
4110 }
4111
4112 static inline void
4113 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4114 {
4115         struct thread *t = &p->threads[p->thread_id];
4116         struct instruction *ip = t->ip;
4117
4118         /* Structs. */
4119         __instr_regprefetch_rm_exec(p, t, ip);
4120
4121         /* Thread. */
4122         thread_ip_inc(p);
4123 }
4124
4125 static inline void
4126 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4127 {
4128         struct thread *t = &p->threads[p->thread_id];
4129         struct instruction *ip = t->ip;
4130
4131         /* Structs. */
4132         __instr_regprefetch_ri_exec(p, t, ip);
4133
4134         /* Thread. */
4135         thread_ip_inc(p);
4136 }
4137
4138 static inline void
4139 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4140 {
4141         struct thread *t = &p->threads[p->thread_id];
4142         struct instruction *ip = t->ip;
4143
4144         /* Structs. */
4145         __instr_regrd_hrh_exec(p, t, ip);
4146
4147         /* Thread. */
4148         thread_ip_inc(p);
4149 }
4150
4151 static inline void
4152 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4153 {
4154         struct thread *t = &p->threads[p->thread_id];
4155         struct instruction *ip = t->ip;
4156
4157         /* Structs. */
4158         __instr_regrd_hrm_exec(p, t, ip);
4159
4160         /* Thread. */
4161         thread_ip_inc(p);
4162 }
4163
4164 static inline void
4165 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4166 {
4167         struct thread *t = &p->threads[p->thread_id];
4168         struct instruction *ip = t->ip;
4169
4170         /* Structs. */
4171         __instr_regrd_mrh_exec(p, t, ip);
4172
4173         /* Thread. */
4174         thread_ip_inc(p);
4175 }
4176
4177 static inline void
4178 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4179 {
4180         struct thread *t = &p->threads[p->thread_id];
4181         struct instruction *ip = t->ip;
4182
4183         /* Structs. */
4184         __instr_regrd_mrm_exec(p, t, ip);
4185
4186         /* Thread. */
4187         thread_ip_inc(p);
4188 }
4189
4190 static inline void
4191 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4192 {
4193         struct thread *t = &p->threads[p->thread_id];
4194         struct instruction *ip = t->ip;
4195
4196         /* Structs. */
4197         __instr_regrd_hri_exec(p, t, ip);
4198
4199         /* Thread. */
4200         thread_ip_inc(p);
4201 }
4202
4203 static inline void
4204 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4205 {
4206         struct thread *t = &p->threads[p->thread_id];
4207         struct instruction *ip = t->ip;
4208
4209         /* Structs. */
4210         __instr_regrd_mri_exec(p, t, ip);
4211
4212         /* Thread. */
4213         thread_ip_inc(p);
4214 }
4215
4216 static inline void
4217 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4218 {
4219         struct thread *t = &p->threads[p->thread_id];
4220         struct instruction *ip = t->ip;
4221
4222         /* Structs. */
4223         __instr_regwr_rhh_exec(p, t, ip);
4224
4225         /* Thread. */
4226         thread_ip_inc(p);
4227 }
4228
4229 static inline void
4230 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4231 {
4232         struct thread *t = &p->threads[p->thread_id];
4233         struct instruction *ip = t->ip;
4234
4235         /* Structs. */
4236         __instr_regwr_rhm_exec(p, t, ip);
4237
4238         /* Thread. */
4239         thread_ip_inc(p);
4240 }
4241
4242 static inline void
4243 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4244 {
4245         struct thread *t = &p->threads[p->thread_id];
4246         struct instruction *ip = t->ip;
4247
4248         /* Structs. */
4249         __instr_regwr_rmh_exec(p, t, ip);
4250
4251         /* Thread. */
4252         thread_ip_inc(p);
4253 }
4254
4255 static inline void
4256 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4257 {
4258         struct thread *t = &p->threads[p->thread_id];
4259         struct instruction *ip = t->ip;
4260
4261         /* Structs. */
4262         __instr_regwr_rmm_exec(p, t, ip);
4263
4264         /* Thread. */
4265         thread_ip_inc(p);
4266 }
4267
4268 static inline void
4269 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4270 {
4271         struct thread *t = &p->threads[p->thread_id];
4272         struct instruction *ip = t->ip;
4273
4274         /* Structs. */
4275         __instr_regwr_rhi_exec(p, t, ip);
4276
4277         /* Thread. */
4278         thread_ip_inc(p);
4279 }
4280
4281 static inline void
4282 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4283 {
4284         struct thread *t = &p->threads[p->thread_id];
4285         struct instruction *ip = t->ip;
4286
4287         /* Structs. */
4288         __instr_regwr_rmi_exec(p, t, ip);
4289
4290         /* Thread. */
4291         thread_ip_inc(p);
4292 }
4293
4294 static inline void
4295 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4296 {
4297         struct thread *t = &p->threads[p->thread_id];
4298         struct instruction *ip = t->ip;
4299
4300         /* Structs. */
4301         __instr_regwr_rih_exec(p, t, ip);
4302
4303         /* Thread. */
4304         thread_ip_inc(p);
4305 }
4306
4307 static inline void
4308 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4309 {
4310         struct thread *t = &p->threads[p->thread_id];
4311         struct instruction *ip = t->ip;
4312
4313         /* Structs. */
4314         __instr_regwr_rim_exec(p, t, ip);
4315
4316         /* Thread. */
4317         thread_ip_inc(p);
4318 }
4319
4320 static inline void
4321 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4322 {
4323         struct thread *t = &p->threads[p->thread_id];
4324         struct instruction *ip = t->ip;
4325
4326         /* Structs. */
4327         __instr_regwr_rii_exec(p, t, ip);
4328
4329         /* Thread. */
4330         thread_ip_inc(p);
4331 }
4332
4333 static inline void
4334 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4335 {
4336         struct thread *t = &p->threads[p->thread_id];
4337         struct instruction *ip = t->ip;
4338
4339         /* Structs. */
4340         __instr_regadd_rhh_exec(p, t, ip);
4341
4342         /* Thread. */
4343         thread_ip_inc(p);
4344 }
4345
4346 static inline void
4347 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4348 {
4349         struct thread *t = &p->threads[p->thread_id];
4350         struct instruction *ip = t->ip;
4351
4352         /* Structs. */
4353         __instr_regadd_rhm_exec(p, t, ip);
4354
4355         /* Thread. */
4356         thread_ip_inc(p);
4357 }
4358
4359 static inline void
4360 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4361 {
4362         struct thread *t = &p->threads[p->thread_id];
4363         struct instruction *ip = t->ip;
4364
4365         /* Structs. */
4366         __instr_regadd_rmh_exec(p, t, ip);
4367
4368         /* Thread. */
4369         thread_ip_inc(p);
4370 }
4371
4372 static inline void
4373 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4374 {
4375         struct thread *t = &p->threads[p->thread_id];
4376         struct instruction *ip = t->ip;
4377
4378         /* Structs. */
4379         __instr_regadd_rmm_exec(p, t, ip);
4380
4381         /* Thread. */
4382         thread_ip_inc(p);
4383 }
4384
4385 static inline void
4386 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4387 {
4388         struct thread *t = &p->threads[p->thread_id];
4389         struct instruction *ip = t->ip;
4390
4391         /* Structs. */
4392         __instr_regadd_rhi_exec(p, t, ip);
4393
4394         /* Thread. */
4395         thread_ip_inc(p);
4396 }
4397
4398 static inline void
4399 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4400 {
4401         struct thread *t = &p->threads[p->thread_id];
4402         struct instruction *ip = t->ip;
4403
4404         /* Structs. */
4405         __instr_regadd_rmi_exec(p, t, ip);
4406
4407         /* Thread. */
4408         thread_ip_inc(p);
4409 }
4410
4411 static inline void
4412 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4413 {
4414         struct thread *t = &p->threads[p->thread_id];
4415         struct instruction *ip = t->ip;
4416
4417         /* Structs. */
4418         __instr_regadd_rih_exec(p, t, ip);
4419
4420         /* Thread. */
4421         thread_ip_inc(p);
4422 }
4423
4424 static inline void
4425 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4426 {
4427         struct thread *t = &p->threads[p->thread_id];
4428         struct instruction *ip = t->ip;
4429
4430         /* Structs. */
4431         __instr_regadd_rim_exec(p, t, ip);
4432
4433         /* Thread. */
4434         thread_ip_inc(p);
4435 }
4436
4437 static inline void
4438 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4439 {
4440         struct thread *t = &p->threads[p->thread_id];
4441         struct instruction *ip = t->ip;
4442
4443         /* Structs. */
4444         __instr_regadd_rii_exec(p, t, ip);
4445
4446         /* Thread. */
4447         thread_ip_inc(p);
4448 }
4449
4450 /*
4451  * metarray.
4452  */
4453 static struct metarray *
4454 metarray_find(struct rte_swx_pipeline *p, const char *name);
4455
4456 static int
4457 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4458                             struct action *action,
4459                             char **tokens,
4460                             int n_tokens,
4461                             struct instruction *instr,
4462                             struct instruction_data *data __rte_unused)
4463 {
4464         char *metarray = tokens[1], *idx = tokens[2];
4465         struct metarray *m;
4466         struct field *fidx;
4467         uint32_t idx_struct_id, idx_val;
4468
4469         CHECK(n_tokens == 3, EINVAL);
4470
4471         m = metarray_find(p, metarray);
4472         CHECK(m, EINVAL);
4473
4474         /* METPREFETCH_H, METPREFETCH_M. */
4475         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4476         if (fidx) {
4477                 CHECK(!fidx->var_size, EINVAL);
4478
4479                 instr->type = INSTR_METPREFETCH_M;
4480                 if (idx[0] == 'h')
4481                         instr->type = INSTR_METPREFETCH_H;
4482
4483                 instr->meter.metarray_id = m->id;
4484                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4485                 instr->meter.idx.n_bits = fidx->n_bits;
4486                 instr->meter.idx.offset = fidx->offset / 8;
4487                 return 0;
4488         }
4489
4490         /* METPREFETCH_I. */
4491         idx_val = strtoul(idx, &idx, 0);
4492         CHECK(!idx[0], EINVAL);
4493
4494         instr->type = INSTR_METPREFETCH_I;
4495         instr->meter.metarray_id = m->id;
4496         instr->meter.idx_val = idx_val;
4497         return 0;
4498 }
4499
4500 static int
4501 instr_meter_translate(struct rte_swx_pipeline *p,
4502                       struct action *action,
4503                       char **tokens,
4504                       int n_tokens,
4505                       struct instruction *instr,
4506                       struct instruction_data *data __rte_unused)
4507 {
4508         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4509         char *color_in = tokens[4], *color_out = tokens[5];
4510         struct metarray *m;
4511         struct field *fidx, *flength, *fcin, *fcout;
4512         uint32_t idx_struct_id, length_struct_id;
4513         uint32_t color_in_struct_id, color_out_struct_id;
4514
4515         CHECK(n_tokens == 6, EINVAL);
4516
4517         m = metarray_find(p, metarray);
4518         CHECK(m, EINVAL);
4519
4520         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4521
4522         flength = struct_field_parse(p, action, length, &length_struct_id);
4523         CHECK(flength, EINVAL);
4524         CHECK(!flength->var_size, EINVAL);
4525
4526         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4527
4528         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4529         CHECK(fcout, EINVAL);
4530         CHECK(!fcout->var_size, EINVAL);
4531
4532         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4533         if (fidx && fcin) {
4534                 CHECK(!fidx->var_size, EINVAL);
4535                 CHECK(!fcin->var_size, EINVAL);
4536
4537                 instr->type = INSTR_METER_MMM;
4538                 if (idx[0] == 'h' && length[0] == 'h')
4539                         instr->type = INSTR_METER_HHM;
4540                 if (idx[0] == 'h' && length[0] != 'h')
4541                         instr->type = INSTR_METER_HMM;
4542                 if (idx[0] != 'h' && length[0] == 'h')
4543                         instr->type = INSTR_METER_MHM;
4544
4545                 instr->meter.metarray_id = m->id;
4546
4547                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4548                 instr->meter.idx.n_bits = fidx->n_bits;
4549                 instr->meter.idx.offset = fidx->offset / 8;
4550
4551                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4552                 instr->meter.length.n_bits = flength->n_bits;
4553                 instr->meter.length.offset = flength->offset / 8;
4554
4555                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4556                 instr->meter.color_in.n_bits = fcin->n_bits;
4557                 instr->meter.color_in.offset = fcin->offset / 8;
4558
4559                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4560                 instr->meter.color_out.n_bits = fcout->n_bits;
4561                 instr->meter.color_out.offset = fcout->offset / 8;
4562         }
4563
4564         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4565         if (fidx && !fcin) {
4566                 uint32_t color_in_val;
4567
4568                 CHECK(!fidx->var_size, EINVAL);
4569
4570                 color_in_val = strtoul(color_in, &color_in, 0);
4571                 CHECK(!color_in[0], EINVAL);
4572
4573                 instr->type = INSTR_METER_MMI;
4574                 if (idx[0] == 'h' && length[0] == 'h')
4575                         instr->type = INSTR_METER_HHI;
4576                 if (idx[0] == 'h' && length[0] != 'h')
4577                         instr->type = INSTR_METER_HMI;
4578                 if (idx[0] != 'h' && length[0] == 'h')
4579                         instr->type = INSTR_METER_MHI;
4580
4581                 instr->meter.metarray_id = m->id;
4582
4583                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4584                 instr->meter.idx.n_bits = fidx->n_bits;
4585                 instr->meter.idx.offset = fidx->offset / 8;
4586
4587                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4588                 instr->meter.length.n_bits = flength->n_bits;
4589                 instr->meter.length.offset = flength->offset / 8;
4590
4591                 instr->meter.color_in_val = color_in_val;
4592
4593                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4594                 instr->meter.color_out.n_bits = fcout->n_bits;
4595                 instr->meter.color_out.offset = fcout->offset / 8;
4596         }
4597
4598         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4599         if (!fidx && fcin) {
4600                 uint32_t idx_val;
4601
4602                 idx_val = strtoul(idx, &idx, 0);
4603                 CHECK(!idx[0], EINVAL);
4604
4605                 CHECK(!fcin->var_size, EINVAL);
4606
4607                 instr->type = INSTR_METER_IMM;
4608                 if (length[0] == 'h')
4609                         instr->type = INSTR_METER_IHM;
4610
4611                 instr->meter.metarray_id = m->id;
4612
4613                 instr->meter.idx_val = idx_val;
4614
4615                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4616                 instr->meter.length.n_bits = flength->n_bits;
4617                 instr->meter.length.offset = flength->offset / 8;
4618
4619                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4620                 instr->meter.color_in.n_bits = fcin->n_bits;
4621                 instr->meter.color_in.offset = fcin->offset / 8;
4622
4623                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4624                 instr->meter.color_out.n_bits = fcout->n_bits;
4625                 instr->meter.color_out.offset = fcout->offset / 8;
4626         }
4627
4628         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4629         if (!fidx && !fcin) {
4630                 uint32_t idx_val, color_in_val;
4631
4632                 idx_val = strtoul(idx, &idx, 0);
4633                 CHECK(!idx[0], EINVAL);
4634
4635                 color_in_val = strtoul(color_in, &color_in, 0);
4636                 CHECK(!color_in[0], EINVAL);
4637
4638                 instr->type = INSTR_METER_IMI;
4639                 if (length[0] == 'h')
4640                         instr->type = INSTR_METER_IHI;
4641
4642                 instr->meter.metarray_id = m->id;
4643
4644                 instr->meter.idx_val = idx_val;
4645
4646                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4647                 instr->meter.length.n_bits = flength->n_bits;
4648                 instr->meter.length.offset = flength->offset / 8;
4649
4650                 instr->meter.color_in_val = color_in_val;
4651
4652                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4653                 instr->meter.color_out.n_bits = fcout->n_bits;
4654                 instr->meter.color_out.offset = fcout->offset / 8;
4655         }
4656
4657         return 0;
4658 }
4659
4660 static inline void
4661 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4662 {
4663         struct thread *t = &p->threads[p->thread_id];
4664         struct instruction *ip = t->ip;
4665
4666         /* Structs. */
4667         __instr_metprefetch_h_exec(p, t, ip);
4668
4669         /* Thread. */
4670         thread_ip_inc(p);
4671 }
4672
4673 static inline void
4674 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4675 {
4676         struct thread *t = &p->threads[p->thread_id];
4677         struct instruction *ip = t->ip;
4678
4679         /* Structs. */
4680         __instr_metprefetch_m_exec(p, t, ip);
4681
4682         /* Thread. */
4683         thread_ip_inc(p);
4684 }
4685
4686 static inline void
4687 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4688 {
4689         struct thread *t = &p->threads[p->thread_id];
4690         struct instruction *ip = t->ip;
4691
4692         /* Structs. */
4693         __instr_metprefetch_i_exec(p, t, ip);
4694
4695         /* Thread. */
4696         thread_ip_inc(p);
4697 }
4698
4699 static inline void
4700 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4701 {
4702         struct thread *t = &p->threads[p->thread_id];
4703         struct instruction *ip = t->ip;
4704
4705         /* Structs. */
4706         __instr_meter_hhm_exec(p, t, ip);
4707
4708         /* Thread. */
4709         thread_ip_inc(p);
4710 }
4711
4712 static inline void
4713 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4714 {
4715         struct thread *t = &p->threads[p->thread_id];
4716         struct instruction *ip = t->ip;
4717
4718         /* Structs. */
4719         __instr_meter_hhi_exec(p, t, ip);
4720
4721         /* Thread. */
4722         thread_ip_inc(p);
4723 }
4724
4725 static inline void
4726 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4727 {
4728         struct thread *t = &p->threads[p->thread_id];
4729         struct instruction *ip = t->ip;
4730
4731         /* Structs. */
4732         __instr_meter_hmm_exec(p, t, ip);
4733
4734         /* Thread. */
4735         thread_ip_inc(p);
4736 }
4737
4738 static inline void
4739 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4740 {
4741         struct thread *t = &p->threads[p->thread_id];
4742         struct instruction *ip = t->ip;
4743
4744         /* Structs. */
4745         __instr_meter_hmi_exec(p, t, ip);
4746
4747         /* Thread. */
4748         thread_ip_inc(p);
4749 }
4750
4751 static inline void
4752 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4753 {
4754         struct thread *t = &p->threads[p->thread_id];
4755         struct instruction *ip = t->ip;
4756
4757         /* Structs. */
4758         __instr_meter_mhm_exec(p, t, ip);
4759
4760         /* Thread. */
4761         thread_ip_inc(p);
4762 }
4763
4764 static inline void
4765 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4766 {
4767         struct thread *t = &p->threads[p->thread_id];
4768         struct instruction *ip = t->ip;
4769
4770         /* Structs. */
4771         __instr_meter_mhi_exec(p, t, ip);
4772
4773         /* Thread. */
4774         thread_ip_inc(p);
4775 }
4776
4777 static inline void
4778 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4779 {
4780         struct thread *t = &p->threads[p->thread_id];
4781         struct instruction *ip = t->ip;
4782
4783         /* Structs. */
4784         __instr_meter_mmm_exec(p, t, ip);
4785
4786         /* Thread. */
4787         thread_ip_inc(p);
4788 }
4789
4790 static inline void
4791 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4792 {
4793         struct thread *t = &p->threads[p->thread_id];
4794         struct instruction *ip = t->ip;
4795
4796         /* Structs. */
4797         __instr_meter_mmi_exec(p, t, ip);
4798
4799         /* Thread. */
4800         thread_ip_inc(p);
4801 }
4802
4803 static inline void
4804 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4805 {
4806         struct thread *t = &p->threads[p->thread_id];
4807         struct instruction *ip = t->ip;
4808
4809         /* Structs. */
4810         __instr_meter_ihm_exec(p, t, ip);
4811
4812         /* Thread. */
4813         thread_ip_inc(p);
4814 }
4815
4816 static inline void
4817 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4818 {
4819         struct thread *t = &p->threads[p->thread_id];
4820         struct instruction *ip = t->ip;
4821
4822         /* Structs. */
4823         __instr_meter_ihi_exec(p, t, ip);
4824
4825         /* Thread. */
4826         thread_ip_inc(p);
4827 }
4828
4829 static inline void
4830 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4831 {
4832         struct thread *t = &p->threads[p->thread_id];
4833         struct instruction *ip = t->ip;
4834
4835         /* Structs. */
4836         __instr_meter_imm_exec(p, t, ip);
4837
4838         /* Thread. */
4839         thread_ip_inc(p);
4840 }
4841
4842 static inline void
4843 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4844 {
4845         struct thread *t = &p->threads[p->thread_id];
4846         struct instruction *ip = t->ip;
4847
4848         /* Structs. */
4849         __instr_meter_imi_exec(p, t, ip);
4850
4851         /* Thread. */
4852         thread_ip_inc(p);
4853 }
4854
4855 /*
4856  * jmp.
4857  */
4858 static int
4859 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4860                     struct action *action __rte_unused,
4861                     char **tokens,
4862                     int n_tokens,
4863                     struct instruction *instr,
4864                     struct instruction_data *data)
4865 {
4866         CHECK(n_tokens == 2, EINVAL);
4867
4868         strcpy(data->jmp_label, tokens[1]);
4869
4870         instr->type = INSTR_JMP;
4871         instr->jmp.ip = NULL; /* Resolved later. */
4872         return 0;
4873 }
4874
4875 static int
4876 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4877                           struct action *action __rte_unused,
4878                           char **tokens,
4879                           int n_tokens,
4880                           struct instruction *instr,
4881                           struct instruction_data *data)
4882 {
4883         struct header *h;
4884
4885         CHECK(n_tokens == 3, EINVAL);
4886
4887         strcpy(data->jmp_label, tokens[1]);
4888
4889         h = header_parse(p, tokens[2]);
4890         CHECK(h, EINVAL);
4891
4892         instr->type = INSTR_JMP_VALID;
4893         instr->jmp.ip = NULL; /* Resolved later. */
4894         instr->jmp.header_id = h->id;
4895         return 0;
4896 }
4897
4898 static int
4899 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4900                             struct action *action __rte_unused,
4901                             char **tokens,
4902                             int n_tokens,
4903                             struct instruction *instr,
4904                             struct instruction_data *data)
4905 {
4906         struct header *h;
4907
4908         CHECK(n_tokens == 3, EINVAL);
4909
4910         strcpy(data->jmp_label, tokens[1]);
4911
4912         h = header_parse(p, tokens[2]);
4913         CHECK(h, EINVAL);
4914
4915         instr->type = INSTR_JMP_INVALID;
4916         instr->jmp.ip = NULL; /* Resolved later. */
4917         instr->jmp.header_id = h->id;
4918         return 0;
4919 }
4920
4921 static int
4922 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4923                         struct action *action,
4924                         char **tokens,
4925                         int n_tokens,
4926                         struct instruction *instr,
4927                         struct instruction_data *data)
4928 {
4929         CHECK(!action, EINVAL);
4930         CHECK(n_tokens == 2, EINVAL);
4931
4932         strcpy(data->jmp_label, tokens[1]);
4933
4934         instr->type = INSTR_JMP_HIT;
4935         instr->jmp.ip = NULL; /* Resolved later. */
4936         return 0;
4937 }
4938
4939 static int
4940 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4941                          struct action *action,
4942                          char **tokens,
4943                          int n_tokens,
4944                          struct instruction *instr,
4945                          struct instruction_data *data)
4946 {
4947         CHECK(!action, EINVAL);
4948         CHECK(n_tokens == 2, EINVAL);
4949
4950         strcpy(data->jmp_label, tokens[1]);
4951
4952         instr->type = INSTR_JMP_MISS;
4953         instr->jmp.ip = NULL; /* Resolved later. */
4954         return 0;
4955 }
4956
4957 static int
4958 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4959                                struct action *action,
4960                                char **tokens,
4961                                int n_tokens,
4962                                struct instruction *instr,
4963                                struct instruction_data *data)
4964 {
4965         struct action *a;
4966
4967         CHECK(!action, EINVAL);
4968         CHECK(n_tokens == 3, EINVAL);
4969
4970         strcpy(data->jmp_label, tokens[1]);
4971
4972         a = action_find(p, tokens[2]);
4973         CHECK(a, EINVAL);
4974
4975         instr->type = INSTR_JMP_ACTION_HIT;
4976         instr->jmp.ip = NULL; /* Resolved later. */
4977         instr->jmp.action_id = a->id;
4978         return 0;
4979 }
4980
4981 static int
4982 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4983                                 struct action *action,
4984                                 char **tokens,
4985                                 int n_tokens,
4986                                 struct instruction *instr,
4987                                 struct instruction_data *data)
4988 {
4989         struct action *a;
4990
4991         CHECK(!action, EINVAL);
4992         CHECK(n_tokens == 3, EINVAL);
4993
4994         strcpy(data->jmp_label, tokens[1]);
4995
4996         a = action_find(p, tokens[2]);
4997         CHECK(a, EINVAL);
4998
4999         instr->type = INSTR_JMP_ACTION_MISS;
5000         instr->jmp.ip = NULL; /* Resolved later. */
5001         instr->jmp.action_id = a->id;
5002         return 0;
5003 }
5004
5005 static int
5006 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5007                        struct action *action,
5008                        char **tokens,
5009                        int n_tokens,
5010                        struct instruction *instr,
5011                        struct instruction_data *data)
5012 {
5013         char *a = tokens[2], *b = tokens[3];
5014         struct field *fa, *fb;
5015         uint64_t b_val;
5016         uint32_t a_struct_id, b_struct_id;
5017
5018         CHECK(n_tokens == 4, EINVAL);
5019
5020         strcpy(data->jmp_label, tokens[1]);
5021
5022         fa = struct_field_parse(p, action, a, &a_struct_id);
5023         CHECK(fa, EINVAL);
5024         CHECK(!fa->var_size, EINVAL);
5025
5026         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5027         fb = struct_field_parse(p, action, b, &b_struct_id);
5028         if (fb) {
5029                 CHECK(!fb->var_size, EINVAL);
5030
5031                 instr->type = INSTR_JMP_EQ;
5032                 if (a[0] != 'h' && b[0] == 'h')
5033                         instr->type = INSTR_JMP_EQ_MH;
5034                 if (a[0] == 'h' && b[0] != 'h')
5035                         instr->type = INSTR_JMP_EQ_HM;
5036                 if (a[0] == 'h' && b[0] == 'h')
5037                         instr->type = INSTR_JMP_EQ_HH;
5038                 instr->jmp.ip = NULL; /* Resolved later. */
5039
5040                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5041                 instr->jmp.a.n_bits = fa->n_bits;
5042                 instr->jmp.a.offset = fa->offset / 8;
5043                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5044                 instr->jmp.b.n_bits = fb->n_bits;
5045                 instr->jmp.b.offset = fb->offset / 8;
5046                 return 0;
5047         }
5048
5049         /* JMP_EQ_I. */
5050         b_val = strtoull(b, &b, 0);
5051         CHECK(!b[0], EINVAL);
5052
5053         if (a[0] == 'h')
5054                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5055
5056         instr->type = INSTR_JMP_EQ_I;
5057         instr->jmp.ip = NULL; /* Resolved later. */
5058         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5059         instr->jmp.a.n_bits = fa->n_bits;
5060         instr->jmp.a.offset = fa->offset / 8;
5061         instr->jmp.b_val = b_val;
5062         return 0;
5063 }
5064
5065 static int
5066 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5067                         struct action *action,
5068                         char **tokens,
5069                         int n_tokens,
5070                         struct instruction *instr,
5071                         struct instruction_data *data)
5072 {
5073         char *a = tokens[2], *b = tokens[3];
5074         struct field *fa, *fb;
5075         uint64_t b_val;
5076         uint32_t a_struct_id, b_struct_id;
5077
5078         CHECK(n_tokens == 4, EINVAL);
5079
5080         strcpy(data->jmp_label, tokens[1]);
5081
5082         fa = struct_field_parse(p, action, a, &a_struct_id);
5083         CHECK(fa, EINVAL);
5084         CHECK(!fa->var_size, EINVAL);
5085
5086         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5087         fb = struct_field_parse(p, action, b, &b_struct_id);
5088         if (fb) {
5089                 CHECK(!fb->var_size, EINVAL);
5090
5091                 instr->type = INSTR_JMP_NEQ;
5092                 if (a[0] != 'h' && b[0] == 'h')
5093                         instr->type = INSTR_JMP_NEQ_MH;
5094                 if (a[0] == 'h' && b[0] != 'h')
5095                         instr->type = INSTR_JMP_NEQ_HM;
5096                 if (a[0] == 'h' && b[0] == 'h')
5097                         instr->type = INSTR_JMP_NEQ_HH;
5098                 instr->jmp.ip = NULL; /* Resolved later. */
5099
5100                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5101                 instr->jmp.a.n_bits = fa->n_bits;
5102                 instr->jmp.a.offset = fa->offset / 8;
5103                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5104                 instr->jmp.b.n_bits = fb->n_bits;
5105                 instr->jmp.b.offset = fb->offset / 8;
5106                 return 0;
5107         }
5108
5109         /* JMP_NEQ_I. */
5110         b_val = strtoull(b, &b, 0);
5111         CHECK(!b[0], EINVAL);
5112
5113         if (a[0] == 'h')
5114                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5115
5116         instr->type = INSTR_JMP_NEQ_I;
5117         instr->jmp.ip = NULL; /* Resolved later. */
5118         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5119         instr->jmp.a.n_bits = fa->n_bits;
5120         instr->jmp.a.offset = fa->offset / 8;
5121         instr->jmp.b_val = b_val;
5122         return 0;
5123 }
5124
5125 static int
5126 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5127                        struct action *action,
5128                        char **tokens,
5129                        int n_tokens,
5130                        struct instruction *instr,
5131                        struct instruction_data *data)
5132 {
5133         char *a = tokens[2], *b = tokens[3];
5134         struct field *fa, *fb;
5135         uint64_t b_val;
5136         uint32_t a_struct_id, b_struct_id;
5137
5138         CHECK(n_tokens == 4, EINVAL);
5139
5140         strcpy(data->jmp_label, tokens[1]);
5141
5142         fa = struct_field_parse(p, action, a, &a_struct_id);
5143         CHECK(fa, EINVAL);
5144         CHECK(!fa->var_size, EINVAL);
5145
5146         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5147         fb = struct_field_parse(p, action, b, &b_struct_id);
5148         if (fb) {
5149                 CHECK(!fb->var_size, EINVAL);
5150
5151                 instr->type = INSTR_JMP_LT;
5152                 if (a[0] == 'h' && b[0] != 'h')
5153                         instr->type = INSTR_JMP_LT_HM;
5154                 if (a[0] != 'h' && b[0] == 'h')
5155                         instr->type = INSTR_JMP_LT_MH;
5156                 if (a[0] == 'h' && b[0] == 'h')
5157                         instr->type = INSTR_JMP_LT_HH;
5158                 instr->jmp.ip = NULL; /* Resolved later. */
5159
5160                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5161                 instr->jmp.a.n_bits = fa->n_bits;
5162                 instr->jmp.a.offset = fa->offset / 8;
5163                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5164                 instr->jmp.b.n_bits = fb->n_bits;
5165                 instr->jmp.b.offset = fb->offset / 8;
5166                 return 0;
5167         }
5168
5169         /* JMP_LT_MI, JMP_LT_HI. */
5170         b_val = strtoull(b, &b, 0);
5171         CHECK(!b[0], EINVAL);
5172
5173         instr->type = INSTR_JMP_LT_MI;
5174         if (a[0] == 'h')
5175                 instr->type = INSTR_JMP_LT_HI;
5176         instr->jmp.ip = NULL; /* Resolved later. */
5177
5178         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5179         instr->jmp.a.n_bits = fa->n_bits;
5180         instr->jmp.a.offset = fa->offset / 8;
5181         instr->jmp.b_val = b_val;
5182         return 0;
5183 }
5184
5185 static int
5186 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5187                        struct action *action,
5188                        char **tokens,
5189                        int n_tokens,
5190                        struct instruction *instr,
5191                        struct instruction_data *data)
5192 {
5193         char *a = tokens[2], *b = tokens[3];
5194         struct field *fa, *fb;
5195         uint64_t b_val;
5196         uint32_t a_struct_id, b_struct_id;
5197
5198         CHECK(n_tokens == 4, EINVAL);
5199
5200         strcpy(data->jmp_label, tokens[1]);
5201
5202         fa = struct_field_parse(p, action, a, &a_struct_id);
5203         CHECK(fa, EINVAL);
5204         CHECK(!fa->var_size, EINVAL);
5205
5206         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5207         fb = struct_field_parse(p, action, b, &b_struct_id);
5208         if (fb) {
5209                 CHECK(!fb->var_size, EINVAL);
5210
5211                 instr->type = INSTR_JMP_GT;
5212                 if (a[0] == 'h' && b[0] != 'h')
5213                         instr->type = INSTR_JMP_GT_HM;
5214                 if (a[0] != 'h' && b[0] == 'h')
5215                         instr->type = INSTR_JMP_GT_MH;
5216                 if (a[0] == 'h' && b[0] == 'h')
5217                         instr->type = INSTR_JMP_GT_HH;
5218                 instr->jmp.ip = NULL; /* Resolved later. */
5219
5220                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5221                 instr->jmp.a.n_bits = fa->n_bits;
5222                 instr->jmp.a.offset = fa->offset / 8;
5223                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5224                 instr->jmp.b.n_bits = fb->n_bits;
5225                 instr->jmp.b.offset = fb->offset / 8;
5226                 return 0;
5227         }
5228
5229         /* JMP_GT_MI, JMP_GT_HI. */
5230         b_val = strtoull(b, &b, 0);
5231         CHECK(!b[0], EINVAL);
5232
5233         instr->type = INSTR_JMP_GT_MI;
5234         if (a[0] == 'h')
5235                 instr->type = INSTR_JMP_GT_HI;
5236         instr->jmp.ip = NULL; /* Resolved later. */
5237
5238         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5239         instr->jmp.a.n_bits = fa->n_bits;
5240         instr->jmp.a.offset = fa->offset / 8;
5241         instr->jmp.b_val = b_val;
5242         return 0;
5243 }
5244
5245 static inline void
5246 instr_jmp_exec(struct rte_swx_pipeline *p)
5247 {
5248         struct thread *t = &p->threads[p->thread_id];
5249         struct instruction *ip = t->ip;
5250
5251         TRACE("[Thread %2u] jmp\n", p->thread_id);
5252
5253         thread_ip_set(t, ip->jmp.ip);
5254 }
5255
5256 static inline void
5257 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5258 {
5259         struct thread *t = &p->threads[p->thread_id];
5260         struct instruction *ip = t->ip;
5261         uint32_t header_id = ip->jmp.header_id;
5262
5263         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5264
5265         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5266 }
5267
5268 static inline void
5269 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5270 {
5271         struct thread *t = &p->threads[p->thread_id];
5272         struct instruction *ip = t->ip;
5273         uint32_t header_id = ip->jmp.header_id;
5274
5275         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5276
5277         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5278 }
5279
5280 static inline void
5281 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5282 {
5283         struct thread *t = &p->threads[p->thread_id];
5284         struct instruction *ip = t->ip;
5285         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5286
5287         TRACE("[Thread %2u] jmph\n", p->thread_id);
5288
5289         t->ip = ip_next[t->hit];
5290 }
5291
5292 static inline void
5293 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5294 {
5295         struct thread *t = &p->threads[p->thread_id];
5296         struct instruction *ip = t->ip;
5297         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5298
5299         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5300
5301         t->ip = ip_next[t->hit];
5302 }
5303
5304 static inline void
5305 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5306 {
5307         struct thread *t = &p->threads[p->thread_id];
5308         struct instruction *ip = t->ip;
5309
5310         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5311
5312         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5313 }
5314
5315 static inline void
5316 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5317 {
5318         struct thread *t = &p->threads[p->thread_id];
5319         struct instruction *ip = t->ip;
5320
5321         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5322
5323         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5324 }
5325
5326 static inline void
5327 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5328 {
5329         struct thread *t = &p->threads[p->thread_id];
5330         struct instruction *ip = t->ip;
5331
5332         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5333
5334         JMP_CMP(t, ip, ==);
5335 }
5336
5337 static inline void
5338 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5339 {
5340         struct thread *t = &p->threads[p->thread_id];
5341         struct instruction *ip = t->ip;
5342
5343         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5344
5345         JMP_CMP_MH(t, ip, ==);
5346 }
5347
5348 static inline void
5349 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5350 {
5351         struct thread *t = &p->threads[p->thread_id];
5352         struct instruction *ip = t->ip;
5353
5354         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5355
5356         JMP_CMP_HM(t, ip, ==);
5357 }
5358
5359 static inline void
5360 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5361 {
5362         struct thread *t = &p->threads[p->thread_id];
5363         struct instruction *ip = t->ip;
5364
5365         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5366
5367         JMP_CMP_HH_FAST(t, ip, ==);
5368 }
5369
5370 static inline void
5371 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5372 {
5373         struct thread *t = &p->threads[p->thread_id];
5374         struct instruction *ip = t->ip;
5375
5376         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5377
5378         JMP_CMP_I(t, ip, ==);
5379 }
5380
5381 static inline void
5382 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5383 {
5384         struct thread *t = &p->threads[p->thread_id];
5385         struct instruction *ip = t->ip;
5386
5387         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5388
5389         JMP_CMP(t, ip, !=);
5390 }
5391
5392 static inline void
5393 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5394 {
5395         struct thread *t = &p->threads[p->thread_id];
5396         struct instruction *ip = t->ip;
5397
5398         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5399
5400         JMP_CMP_MH(t, ip, !=);
5401 }
5402
5403 static inline void
5404 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5405 {
5406         struct thread *t = &p->threads[p->thread_id];
5407         struct instruction *ip = t->ip;
5408
5409         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5410
5411         JMP_CMP_HM(t, ip, !=);
5412 }
5413
5414 static inline void
5415 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5416 {
5417         struct thread *t = &p->threads[p->thread_id];
5418         struct instruction *ip = t->ip;
5419
5420         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5421
5422         JMP_CMP_HH_FAST(t, ip, !=);
5423 }
5424
5425 static inline void
5426 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5427 {
5428         struct thread *t = &p->threads[p->thread_id];
5429         struct instruction *ip = t->ip;
5430
5431         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5432
5433         JMP_CMP_I(t, ip, !=);
5434 }
5435
5436 static inline void
5437 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5438 {
5439         struct thread *t = &p->threads[p->thread_id];
5440         struct instruction *ip = t->ip;
5441
5442         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5443
5444         JMP_CMP(t, ip, <);
5445 }
5446
5447 static inline void
5448 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5449 {
5450         struct thread *t = &p->threads[p->thread_id];
5451         struct instruction *ip = t->ip;
5452
5453         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5454
5455         JMP_CMP_MH(t, ip, <);
5456 }
5457
5458 static inline void
5459 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5460 {
5461         struct thread *t = &p->threads[p->thread_id];
5462         struct instruction *ip = t->ip;
5463
5464         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5465
5466         JMP_CMP_HM(t, ip, <);
5467 }
5468
5469 static inline void
5470 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5471 {
5472         struct thread *t = &p->threads[p->thread_id];
5473         struct instruction *ip = t->ip;
5474
5475         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5476
5477         JMP_CMP_HH(t, ip, <);
5478 }
5479
5480 static inline void
5481 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5482 {
5483         struct thread *t = &p->threads[p->thread_id];
5484         struct instruction *ip = t->ip;
5485
5486         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5487
5488         JMP_CMP_MI(t, ip, <);
5489 }
5490
5491 static inline void
5492 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5493 {
5494         struct thread *t = &p->threads[p->thread_id];
5495         struct instruction *ip = t->ip;
5496
5497         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5498
5499         JMP_CMP_HI(t, ip, <);
5500 }
5501
5502 static inline void
5503 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5504 {
5505         struct thread *t = &p->threads[p->thread_id];
5506         struct instruction *ip = t->ip;
5507
5508         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5509
5510         JMP_CMP(t, ip, >);
5511 }
5512
5513 static inline void
5514 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5515 {
5516         struct thread *t = &p->threads[p->thread_id];
5517         struct instruction *ip = t->ip;
5518
5519         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5520
5521         JMP_CMP_MH(t, ip, >);
5522 }
5523
5524 static inline void
5525 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5526 {
5527         struct thread *t = &p->threads[p->thread_id];
5528         struct instruction *ip = t->ip;
5529
5530         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5531
5532         JMP_CMP_HM(t, ip, >);
5533 }
5534
5535 static inline void
5536 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5537 {
5538         struct thread *t = &p->threads[p->thread_id];
5539         struct instruction *ip = t->ip;
5540
5541         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5542
5543         JMP_CMP_HH(t, ip, >);
5544 }
5545
5546 static inline void
5547 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5548 {
5549         struct thread *t = &p->threads[p->thread_id];
5550         struct instruction *ip = t->ip;
5551
5552         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5553
5554         JMP_CMP_MI(t, ip, >);
5555 }
5556
5557 static inline void
5558 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5559 {
5560         struct thread *t = &p->threads[p->thread_id];
5561         struct instruction *ip = t->ip;
5562
5563         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5564
5565         JMP_CMP_HI(t, ip, >);
5566 }
5567
5568 /*
5569  * return.
5570  */
5571 static int
5572 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5573                        struct action *action,
5574                        char **tokens __rte_unused,
5575                        int n_tokens,
5576                        struct instruction *instr,
5577                        struct instruction_data *data __rte_unused)
5578 {
5579         CHECK(action, EINVAL);
5580         CHECK(n_tokens == 1, EINVAL);
5581
5582         instr->type = INSTR_RETURN;
5583         return 0;
5584 }
5585
5586 static inline void
5587 instr_return_exec(struct rte_swx_pipeline *p)
5588 {
5589         struct thread *t = &p->threads[p->thread_id];
5590
5591         TRACE("[Thread %2u] return\n", p->thread_id);
5592
5593         t->ip = t->ret;
5594 }
5595
5596 static int
5597 instr_translate(struct rte_swx_pipeline *p,
5598                 struct action *action,
5599                 char *string,
5600                 struct instruction *instr,
5601                 struct instruction_data *data)
5602 {
5603         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5604         int n_tokens = 0, tpos = 0;
5605
5606         /* Parse the instruction string into tokens. */
5607         for ( ; ; ) {
5608                 char *token;
5609
5610                 token = strtok_r(string, " \t\v", &string);
5611                 if (!token)
5612                         break;
5613
5614                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5615                 CHECK_NAME(token, EINVAL);
5616
5617                 tokens[n_tokens] = token;
5618                 n_tokens++;
5619         }
5620
5621         CHECK(n_tokens, EINVAL);
5622
5623         /* Handle the optional instruction label. */
5624         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5625                 strcpy(data->label, tokens[0]);
5626
5627                 tpos += 2;
5628                 CHECK(n_tokens - tpos, EINVAL);
5629         }
5630
5631         /* Identify the instruction type. */
5632         if (!strcmp(tokens[tpos], "rx"))
5633                 return instr_rx_translate(p,
5634                                           action,
5635                                           &tokens[tpos],
5636                                           n_tokens - tpos,
5637                                           instr,
5638                                           data);
5639
5640         if (!strcmp(tokens[tpos], "tx"))
5641                 return instr_tx_translate(p,
5642                                           action,
5643                                           &tokens[tpos],
5644                                           n_tokens - tpos,
5645                                           instr,
5646                                           data);
5647
5648         if (!strcmp(tokens[tpos], "drop"))
5649                 return instr_drop_translate(p,
5650                                             action,
5651                                             &tokens[tpos],
5652                                             n_tokens - tpos,
5653                                             instr,
5654                                             data);
5655
5656         if (!strcmp(tokens[tpos], "extract"))
5657                 return instr_hdr_extract_translate(p,
5658                                                    action,
5659                                                    &tokens[tpos],
5660                                                    n_tokens - tpos,
5661                                                    instr,
5662                                                    data);
5663
5664         if (!strcmp(tokens[tpos], "lookahead"))
5665                 return instr_hdr_lookahead_translate(p,
5666                                                      action,
5667                                                      &tokens[tpos],
5668                                                      n_tokens - tpos,
5669                                                      instr,
5670                                                      data);
5671
5672         if (!strcmp(tokens[tpos], "emit"))
5673                 return instr_hdr_emit_translate(p,
5674                                                 action,
5675                                                 &tokens[tpos],
5676                                                 n_tokens - tpos,
5677                                                 instr,
5678                                                 data);
5679
5680         if (!strcmp(tokens[tpos], "validate"))
5681                 return instr_hdr_validate_translate(p,
5682                                                     action,
5683                                                     &tokens[tpos],
5684                                                     n_tokens - tpos,
5685                                                     instr,
5686                                                     data);
5687
5688         if (!strcmp(tokens[tpos], "invalidate"))
5689                 return instr_hdr_invalidate_translate(p,
5690                                                       action,
5691                                                       &tokens[tpos],
5692                                                       n_tokens - tpos,
5693                                                       instr,
5694                                                       data);
5695
5696         if (!strcmp(tokens[tpos], "mov"))
5697                 return instr_mov_translate(p,
5698                                            action,
5699                                            &tokens[tpos],
5700                                            n_tokens - tpos,
5701                                            instr,
5702                                            data);
5703
5704         if (!strcmp(tokens[tpos], "add"))
5705                 return instr_alu_add_translate(p,
5706                                                action,
5707                                                &tokens[tpos],
5708                                                n_tokens - tpos,
5709                                                instr,
5710                                                data);
5711
5712         if (!strcmp(tokens[tpos], "sub"))
5713                 return instr_alu_sub_translate(p,
5714                                                action,
5715                                                &tokens[tpos],
5716                                                n_tokens - tpos,
5717                                                instr,
5718                                                data);
5719
5720         if (!strcmp(tokens[tpos], "ckadd"))
5721                 return instr_alu_ckadd_translate(p,
5722                                                  action,
5723                                                  &tokens[tpos],
5724                                                  n_tokens - tpos,
5725                                                  instr,
5726                                                  data);
5727
5728         if (!strcmp(tokens[tpos], "cksub"))
5729                 return instr_alu_cksub_translate(p,
5730                                                  action,
5731                                                  &tokens[tpos],
5732                                                  n_tokens - tpos,
5733                                                  instr,
5734                                                  data);
5735
5736         if (!strcmp(tokens[tpos], "and"))
5737                 return instr_alu_and_translate(p,
5738                                                action,
5739                                                &tokens[tpos],
5740                                                n_tokens - tpos,
5741                                                instr,
5742                                                data);
5743
5744         if (!strcmp(tokens[tpos], "or"))
5745                 return instr_alu_or_translate(p,
5746                                               action,
5747                                               &tokens[tpos],
5748                                               n_tokens - tpos,
5749                                               instr,
5750                                               data);
5751
5752         if (!strcmp(tokens[tpos], "xor"))
5753                 return instr_alu_xor_translate(p,
5754                                                action,
5755                                                &tokens[tpos],
5756                                                n_tokens - tpos,
5757                                                instr,
5758                                                data);
5759
5760         if (!strcmp(tokens[tpos], "shl"))
5761                 return instr_alu_shl_translate(p,
5762                                                action,
5763                                                &tokens[tpos],
5764                                                n_tokens - tpos,
5765                                                instr,
5766                                                data);
5767
5768         if (!strcmp(tokens[tpos], "shr"))
5769                 return instr_alu_shr_translate(p,
5770                                                action,
5771                                                &tokens[tpos],
5772                                                n_tokens - tpos,
5773                                                instr,
5774                                                data);
5775
5776         if (!strcmp(tokens[tpos], "regprefetch"))
5777                 return instr_regprefetch_translate(p,
5778                                                    action,
5779                                                    &tokens[tpos],
5780                                                    n_tokens - tpos,
5781                                                    instr,
5782                                                    data);
5783
5784         if (!strcmp(tokens[tpos], "regrd"))
5785                 return instr_regrd_translate(p,
5786                                              action,
5787                                              &tokens[tpos],
5788                                              n_tokens - tpos,
5789                                              instr,
5790                                              data);
5791
5792         if (!strcmp(tokens[tpos], "regwr"))
5793                 return instr_regwr_translate(p,
5794                                              action,
5795                                              &tokens[tpos],
5796                                              n_tokens - tpos,
5797                                              instr,
5798                                              data);
5799
5800         if (!strcmp(tokens[tpos], "regadd"))
5801                 return instr_regadd_translate(p,
5802                                               action,
5803                                               &tokens[tpos],
5804                                               n_tokens - tpos,
5805                                               instr,
5806                                               data);
5807
5808         if (!strcmp(tokens[tpos], "metprefetch"))
5809                 return instr_metprefetch_translate(p,
5810                                                    action,
5811                                                    &tokens[tpos],
5812                                                    n_tokens - tpos,
5813                                                    instr,
5814                                                    data);
5815
5816         if (!strcmp(tokens[tpos], "meter"))
5817                 return instr_meter_translate(p,
5818                                              action,
5819                                              &tokens[tpos],
5820                                              n_tokens - tpos,
5821                                              instr,
5822                                              data);
5823
5824         if (!strcmp(tokens[tpos], "table"))
5825                 return instr_table_translate(p,
5826                                              action,
5827                                              &tokens[tpos],
5828                                              n_tokens - tpos,
5829                                              instr,
5830                                              data);
5831
5832         if (!strcmp(tokens[tpos], "learn"))
5833                 return instr_learn_translate(p,
5834                                              action,
5835                                              &tokens[tpos],
5836                                              n_tokens - tpos,
5837                                              instr,
5838                                              data);
5839
5840         if (!strcmp(tokens[tpos], "forget"))
5841                 return instr_forget_translate(p,
5842                                               action,
5843                                               &tokens[tpos],
5844                                               n_tokens - tpos,
5845                                               instr,
5846                                               data);
5847
5848         if (!strcmp(tokens[tpos], "extern"))
5849                 return instr_extern_translate(p,
5850                                               action,
5851                                               &tokens[tpos],
5852                                               n_tokens - tpos,
5853                                               instr,
5854                                               data);
5855
5856         if (!strcmp(tokens[tpos], "jmp"))
5857                 return instr_jmp_translate(p,
5858                                            action,
5859                                            &tokens[tpos],
5860                                            n_tokens - tpos,
5861                                            instr,
5862                                            data);
5863
5864         if (!strcmp(tokens[tpos], "jmpv"))
5865                 return instr_jmp_valid_translate(p,
5866                                                  action,
5867                                                  &tokens[tpos],
5868                                                  n_tokens - tpos,
5869                                                  instr,
5870                                                  data);
5871
5872         if (!strcmp(tokens[tpos], "jmpnv"))
5873                 return instr_jmp_invalid_translate(p,
5874                                                    action,
5875                                                    &tokens[tpos],
5876                                                    n_tokens - tpos,
5877                                                    instr,
5878                                                    data);
5879
5880         if (!strcmp(tokens[tpos], "jmph"))
5881                 return instr_jmp_hit_translate(p,
5882                                                action,
5883                                                &tokens[tpos],
5884                                                n_tokens - tpos,
5885                                                instr,
5886                                                data);
5887
5888         if (!strcmp(tokens[tpos], "jmpnh"))
5889                 return instr_jmp_miss_translate(p,
5890                                                 action,
5891                                                 &tokens[tpos],
5892                                                 n_tokens - tpos,
5893                                                 instr,
5894                                                 data);
5895
5896         if (!strcmp(tokens[tpos], "jmpa"))
5897                 return instr_jmp_action_hit_translate(p,
5898                                                       action,
5899                                                       &tokens[tpos],
5900                                                       n_tokens - tpos,
5901                                                       instr,
5902                                                       data);
5903
5904         if (!strcmp(tokens[tpos], "jmpna"))
5905                 return instr_jmp_action_miss_translate(p,
5906                                                        action,
5907                                                        &tokens[tpos],
5908                                                        n_tokens - tpos,
5909                                                        instr,
5910                                                        data);
5911
5912         if (!strcmp(tokens[tpos], "jmpeq"))
5913                 return instr_jmp_eq_translate(p,
5914                                               action,
5915                                               &tokens[tpos],
5916                                               n_tokens - tpos,
5917                                               instr,
5918                                               data);
5919
5920         if (!strcmp(tokens[tpos], "jmpneq"))
5921                 return instr_jmp_neq_translate(p,
5922                                                action,
5923                                                &tokens[tpos],
5924                                                n_tokens - tpos,
5925                                                instr,
5926                                                data);
5927
5928         if (!strcmp(tokens[tpos], "jmplt"))
5929                 return instr_jmp_lt_translate(p,
5930                                               action,
5931                                               &tokens[tpos],
5932                                               n_tokens - tpos,
5933                                               instr,
5934                                               data);
5935
5936         if (!strcmp(tokens[tpos], "jmpgt"))
5937                 return instr_jmp_gt_translate(p,
5938                                               action,
5939                                               &tokens[tpos],
5940                                               n_tokens - tpos,
5941                                               instr,
5942                                               data);
5943
5944         if (!strcmp(tokens[tpos], "return"))
5945                 return instr_return_translate(p,
5946                                               action,
5947                                               &tokens[tpos],
5948                                               n_tokens - tpos,
5949                                               instr,
5950                                               data);
5951
5952         return -EINVAL;
5953 }
5954
5955 static struct instruction_data *
5956 label_find(struct instruction_data *data, uint32_t n, const char *label)
5957 {
5958         uint32_t i;
5959
5960         for (i = 0; i < n; i++)
5961                 if (!strcmp(label, data[i].label))
5962                         return &data[i];
5963
5964         return NULL;
5965 }
5966
5967 static uint32_t
5968 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5969 {
5970         uint32_t count = 0, i;
5971
5972         if (!label[0])
5973                 return 0;
5974
5975         for (i = 0; i < n; i++)
5976                 if (!strcmp(label, data[i].jmp_label))
5977                         count++;
5978
5979         return count;
5980 }
5981
5982 static int
5983 instr_label_check(struct instruction_data *instruction_data,
5984                   uint32_t n_instructions)
5985 {
5986         uint32_t i;
5987
5988         /* Check that all instruction labels are unique. */
5989         for (i = 0; i < n_instructions; i++) {
5990                 struct instruction_data *data = &instruction_data[i];
5991                 char *label = data->label;
5992                 uint32_t j;
5993
5994                 if (!label[0])
5995                         continue;
5996
5997                 for (j = i + 1; j < n_instructions; j++)
5998                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
5999         }
6000
6001         /* Check that no jump instruction (either conditional or not) can jump to itself (loop). */
6002         for (i = 0; i < n_instructions; i++) {
6003                 struct instruction_data *data = &instruction_data[i];
6004                 char *label = data->label;
6005                 char *jmp_label = data->jmp_label;
6006
6007                 /* Continue if this instruction does not have a label or it is not a jump. */
6008                 if (!label[0] || !jmp_label[0])
6009                         continue;
6010
6011                 CHECK(strcmp(label, jmp_label), EINVAL);
6012         }
6013
6014         /* Get users for each instruction label. */
6015         for (i = 0; i < n_instructions; i++) {
6016                 struct instruction_data *data = &instruction_data[i];
6017                 char *label = data->label;
6018
6019                 data->n_users = label_is_used(instruction_data,
6020                                               n_instructions,
6021                                               label);
6022         }
6023
6024         return 0;
6025 }
6026
6027 static int
6028 instr_jmp_resolve(struct instruction *instructions,
6029                   struct instruction_data *instruction_data,
6030                   uint32_t n_instructions)
6031 {
6032         uint32_t i;
6033
6034         for (i = 0; i < n_instructions; i++) {
6035                 struct instruction *instr = &instructions[i];
6036                 struct instruction_data *data = &instruction_data[i];
6037                 struct instruction_data *found;
6038
6039                 if (!instruction_is_jmp(instr))
6040                         continue;
6041
6042                 found = label_find(instruction_data,
6043                                    n_instructions,
6044                                    data->jmp_label);
6045                 CHECK(found, EINVAL);
6046
6047                 instr->jmp.ip = &instructions[found - instruction_data];
6048         }
6049
6050         return 0;
6051 }
6052
6053 static int
6054 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6055              struct action *a,
6056              struct instruction *instr,
6057              struct instruction_data *data __rte_unused,
6058              uint32_t n_instructions)
6059 {
6060         if (!a) {
6061                 enum instruction_type type;
6062                 uint32_t i;
6063
6064                 /* Check that the first instruction is rx. */
6065                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6066
6067                 /* Check that there is at least one tx instruction. */
6068                 for (i = 0; i < n_instructions; i++) {
6069                         type = instr[i].type;
6070
6071                         if (instruction_is_tx(type))
6072                                 break;
6073                 }
6074                 CHECK(i < n_instructions, EINVAL);
6075
6076                 /* Check that the last instruction is either tx or unconditional
6077                  * jump.
6078                  */
6079                 type = instr[n_instructions - 1].type;
6080                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6081         }
6082
6083         if (a) {
6084                 enum instruction_type type;
6085                 uint32_t i;
6086
6087                 /* Check that there is at least one return or tx instruction. */
6088                 for (i = 0; i < n_instructions; i++) {
6089                         type = instr[i].type;
6090
6091                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6092                                 break;
6093                 }
6094                 CHECK(i < n_instructions, EINVAL);
6095         }
6096
6097         return 0;
6098 }
6099
6100 static uint32_t
6101 instr_compact(struct instruction *instructions,
6102               struct instruction_data *instruction_data,
6103               uint32_t n_instructions)
6104 {
6105         uint32_t i, pos = 0;
6106
6107         /* Eliminate the invalid instructions that have been optimized out. */
6108         for (i = 0; i < n_instructions; i++) {
6109                 struct instruction *instr = &instructions[i];
6110                 struct instruction_data *data = &instruction_data[i];
6111
6112                 if (data->invalid)
6113                         continue;
6114
6115                 if (i != pos) {
6116                         memcpy(&instructions[pos], instr, sizeof(*instr));
6117                         memcpy(&instruction_data[pos], data, sizeof(*data));
6118                 }
6119
6120                 pos++;
6121         }
6122
6123         return pos;
6124 }
6125
6126 static int
6127 instr_pattern_extract_many_search(struct instruction *instr,
6128                                   struct instruction_data *data,
6129                                   uint32_t n_instr,
6130                                   uint32_t *n_pattern_instr)
6131 {
6132         uint32_t i;
6133
6134         for (i = 0; i < n_instr; i++) {
6135                 if (data[i].invalid)
6136                         break;
6137
6138                 if (instr[i].type != INSTR_HDR_EXTRACT)
6139                         break;
6140
6141                 if (i == RTE_DIM(instr->io.hdr.header_id))
6142                         break;
6143
6144                 if (i && data[i].n_users)
6145                         break;
6146         }
6147
6148         if (i < 2)
6149                 return 0;
6150
6151         *n_pattern_instr = i;
6152         return 1;
6153 }
6154
6155 static void
6156 instr_pattern_extract_many_replace(struct instruction *instr,
6157                                    struct instruction_data *data,
6158                                    uint32_t n_instr)
6159 {
6160         uint32_t i;
6161
6162         for (i = 1; i < n_instr; i++) {
6163                 instr[0].type++;
6164                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6165                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6166                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6167
6168                 data[i].invalid = 1;
6169         }
6170 }
6171
6172 static uint32_t
6173 instr_pattern_extract_many_optimize(struct instruction *instructions,
6174                                     struct instruction_data *instruction_data,
6175                                     uint32_t n_instructions)
6176 {
6177         uint32_t i;
6178
6179         for (i = 0; i < n_instructions; ) {
6180                 struct instruction *instr = &instructions[i];
6181                 struct instruction_data *data = &instruction_data[i];
6182                 uint32_t n_instr = 0;
6183                 int detected;
6184
6185                 /* Extract many. */
6186                 detected = instr_pattern_extract_many_search(instr,
6187                                                              data,
6188                                                              n_instructions - i,
6189                                                              &n_instr);
6190                 if (detected) {
6191                         instr_pattern_extract_many_replace(instr,
6192                                                            data,
6193                                                            n_instr);
6194                         i += n_instr;
6195                         continue;
6196                 }
6197
6198                 /* No pattern starting at the current instruction. */
6199                 i++;
6200         }
6201
6202         /* Eliminate the invalid instructions that have been optimized out. */
6203         n_instructions = instr_compact(instructions,
6204                                        instruction_data,
6205                                        n_instructions);
6206
6207         return n_instructions;
6208 }
6209
6210 static int
6211 instr_pattern_emit_many_tx_search(struct instruction *instr,
6212                                   struct instruction_data *data,
6213                                   uint32_t n_instr,
6214                                   uint32_t *n_pattern_instr)
6215 {
6216         uint32_t i;
6217
6218         for (i = 0; i < n_instr; i++) {
6219                 if (data[i].invalid)
6220                         break;
6221
6222                 if (instr[i].type != INSTR_HDR_EMIT)
6223                         break;
6224
6225                 if (i == RTE_DIM(instr->io.hdr.header_id))
6226                         break;
6227
6228                 if (i && data[i].n_users)
6229                         break;
6230         }
6231
6232         if (!i)
6233                 return 0;
6234
6235         if (instr[i].type != INSTR_TX)
6236                 return 0;
6237
6238         if (data[i].n_users)
6239                 return 0;
6240
6241         i++;
6242
6243         *n_pattern_instr = i;
6244         return 1;
6245 }
6246
6247 static void
6248 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6249                                    struct instruction_data *data,
6250                                    uint32_t n_instr)
6251 {
6252         uint32_t i;
6253
6254         /* Any emit instruction in addition to the first one. */
6255         for (i = 1; i < n_instr - 1; i++) {
6256                 instr[0].type++;
6257                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6258                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6259                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6260
6261                 data[i].invalid = 1;
6262         }
6263
6264         /* The TX instruction is the last one in the pattern. */
6265         instr[0].type++;
6266         instr[0].io.io.offset = instr[i].io.io.offset;
6267         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6268         data[i].invalid = 1;
6269 }
6270
6271 static uint32_t
6272 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6273                                     struct instruction_data *instruction_data,
6274                                     uint32_t n_instructions)
6275 {
6276         uint32_t i;
6277
6278         for (i = 0; i < n_instructions; ) {
6279                 struct instruction *instr = &instructions[i];
6280                 struct instruction_data *data = &instruction_data[i];
6281                 uint32_t n_instr = 0;
6282                 int detected;
6283
6284                 /* Emit many + TX. */
6285                 detected = instr_pattern_emit_many_tx_search(instr,
6286                                                              data,
6287                                                              n_instructions - i,
6288                                                              &n_instr);
6289                 if (detected) {
6290                         instr_pattern_emit_many_tx_replace(instr,
6291                                                            data,
6292                                                            n_instr);
6293                         i += n_instr;
6294                         continue;
6295                 }
6296
6297                 /* No pattern starting at the current instruction. */
6298                 i++;
6299         }
6300
6301         /* Eliminate the invalid instructions that have been optimized out. */
6302         n_instructions = instr_compact(instructions,
6303                                        instruction_data,
6304                                        n_instructions);
6305
6306         return n_instructions;
6307 }
6308
6309 static uint32_t
6310 action_arg_src_mov_count(struct action *a,
6311                          uint32_t arg_id,
6312                          struct instruction *instructions,
6313                          struct instruction_data *instruction_data,
6314                          uint32_t n_instructions);
6315
6316 static int
6317 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6318                                       struct action *a,
6319                                       struct instruction *instr,
6320                                       struct instruction_data *data,
6321                                       uint32_t n_instr,
6322                                       struct instruction *instructions,
6323                                       struct instruction_data *instruction_data,
6324                                       uint32_t n_instructions,
6325                                       uint32_t *n_pattern_instr)
6326 {
6327         struct header *h;
6328         uint32_t src_field_id, i, j;
6329
6330         /* Prerequisites. */
6331         if (!a || !a->st)
6332                 return 0;
6333
6334         /* First instruction: MOV_HM. */
6335         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6336                 return 0;
6337
6338         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6339         if (!h || h->st->var_size)
6340                 return 0;
6341
6342         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6343                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6344                         break;
6345
6346         if (src_field_id == a->st->n_fields)
6347                 return 0;
6348
6349         if (instr[0].mov.dst.offset ||
6350             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6351             instr[0].mov.src.struct_id ||
6352             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6353             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6354                 return 0;
6355
6356         if ((n_instr < h->st->n_fields + 1) ||
6357              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6358                 return 0;
6359
6360         /* Subsequent instructions: MOV_HM. */
6361         for (i = 1; i < h->st->n_fields; i++)
6362                 if (data[i].invalid ||
6363                     data[i].n_users ||
6364                     (instr[i].type != INSTR_MOV_HM) ||
6365                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6366                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6367                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6368                     instr[i].mov.src.struct_id ||
6369                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6370                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6371                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6372                         return 0;
6373
6374         /* Last instruction: HDR_VALIDATE. */
6375         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6376             (instr[i].valid.header_id != h->id))
6377                 return 0;
6378
6379         /* Check that none of the action args that are used as source for this
6380          * DMA transfer are not used as source in any other mov instruction.
6381          */
6382         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6383                 uint32_t n_users;
6384
6385                 n_users = action_arg_src_mov_count(a,
6386                                                    j,
6387                                                    instructions,
6388                                                    instruction_data,
6389                                                    n_instructions);
6390                 if (n_users > 1)
6391                         return 0;
6392         }
6393
6394         *n_pattern_instr = 1 + i;
6395         return 1;
6396 }
6397
6398 static void
6399 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6400                                        struct action *a,
6401                                        struct instruction *instr,
6402                                        struct instruction_data *data,
6403                                        uint32_t n_instr)
6404 {
6405         struct header *h;
6406         uint32_t src_field_id, src_offset, i;
6407
6408         /* Read from the instructions before they are modified. */
6409         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6410         if (!h)
6411                 return;
6412
6413         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6414                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6415                         break;
6416
6417         if (src_field_id == a->st->n_fields)
6418                 return;
6419
6420         src_offset = instr[0].mov.src.offset;
6421
6422         /* Modify the instructions. */
6423         instr[0].type = INSTR_DMA_HT;
6424         instr[0].dma.dst.header_id[0] = h->id;
6425         instr[0].dma.dst.struct_id[0] = h->struct_id;
6426         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6427         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6428
6429         for (i = 1; i < n_instr; i++)
6430                 data[i].invalid = 1;
6431
6432         /* Update the endianness of the action arguments to header endianness. */
6433         for (i = 0; i < h->st->n_fields; i++)
6434                 a->args_endianness[src_field_id + i] = 1;
6435 }
6436
6437 static uint32_t
6438 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6439                                         struct action *a,
6440                                         struct instruction *instructions,
6441                                         struct instruction_data *instruction_data,
6442                                         uint32_t n_instructions)
6443 {
6444         uint32_t i;
6445
6446         if (!a || !a->st)
6447                 return n_instructions;
6448
6449         for (i = 0; i < n_instructions; ) {
6450                 struct instruction *instr = &instructions[i];
6451                 struct instruction_data *data = &instruction_data[i];
6452                 uint32_t n_instr = 0;
6453                 int detected;
6454
6455                 /* Mov all + validate. */
6456                 detected = instr_pattern_mov_all_validate_search(p,
6457                                                                  a,
6458                                                                  instr,
6459                                                                  data,
6460                                                                  n_instructions - i,
6461                                                                  instructions,
6462                                                                  instruction_data,
6463                                                                  n_instructions,
6464                                                                  &n_instr);
6465                 if (detected) {
6466                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6467                         i += n_instr;
6468                         continue;
6469                 }
6470
6471                 /* No pattern starting at the current instruction. */
6472                 i++;
6473         }
6474
6475         /* Eliminate the invalid instructions that have been optimized out. */
6476         n_instructions = instr_compact(instructions,
6477                                        instruction_data,
6478                                        n_instructions);
6479
6480         return n_instructions;
6481 }
6482
6483 static int
6484 instr_pattern_dma_many_search(struct instruction *instr,
6485                               struct instruction_data *data,
6486                               uint32_t n_instr,
6487                               uint32_t *n_pattern_instr)
6488 {
6489         uint32_t i;
6490
6491         for (i = 0; i < n_instr; i++) {
6492                 if (data[i].invalid)
6493                         break;
6494
6495                 if (instr[i].type != INSTR_DMA_HT)
6496                         break;
6497
6498                 if (i == RTE_DIM(instr->dma.dst.header_id))
6499                         break;
6500
6501                 if (i && data[i].n_users)
6502                         break;
6503         }
6504
6505         if (i < 2)
6506                 return 0;
6507
6508         *n_pattern_instr = i;
6509         return 1;
6510 }
6511
6512 static void
6513 instr_pattern_dma_many_replace(struct instruction *instr,
6514                                struct instruction_data *data,
6515                                uint32_t n_instr)
6516 {
6517         uint32_t i;
6518
6519         for (i = 1; i < n_instr; i++) {
6520                 instr[0].type++;
6521                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6522                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6523                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6524                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6525
6526                 data[i].invalid = 1;
6527         }
6528 }
6529
6530 static uint32_t
6531 instr_pattern_dma_many_optimize(struct instruction *instructions,
6532                struct instruction_data *instruction_data,
6533                uint32_t n_instructions)
6534 {
6535         uint32_t i;
6536
6537         for (i = 0; i < n_instructions; ) {
6538                 struct instruction *instr = &instructions[i];
6539                 struct instruction_data *data = &instruction_data[i];
6540                 uint32_t n_instr = 0;
6541                 int detected;
6542
6543                 /* DMA many. */
6544                 detected = instr_pattern_dma_many_search(instr,
6545                                                          data,
6546                                                          n_instructions - i,
6547                                                          &n_instr);
6548                 if (detected) {
6549                         instr_pattern_dma_many_replace(instr, data, n_instr);
6550                         i += n_instr;
6551                         continue;
6552                 }
6553
6554                 /* No pattern starting at the current instruction. */
6555                 i++;
6556         }
6557
6558         /* Eliminate the invalid instructions that have been optimized out. */
6559         n_instructions = instr_compact(instructions,
6560                                        instruction_data,
6561                                        n_instructions);
6562
6563         return n_instructions;
6564 }
6565
6566 static uint32_t
6567 instr_optimize(struct rte_swx_pipeline *p,
6568                struct action *a,
6569                struct instruction *instructions,
6570                struct instruction_data *instruction_data,
6571                uint32_t n_instructions)
6572 {
6573         /* Extract many. */
6574         n_instructions = instr_pattern_extract_many_optimize(instructions,
6575                                                              instruction_data,
6576                                                              n_instructions);
6577
6578         /* Emit many + TX. */
6579         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6580                                                              instruction_data,
6581                                                              n_instructions);
6582
6583         /* Mov all + validate. */
6584         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6585                                                                  a,
6586                                                                  instructions,
6587                                                                  instruction_data,
6588                                                                  n_instructions);
6589
6590         /* DMA many. */
6591         n_instructions = instr_pattern_dma_many_optimize(instructions,
6592                                                          instruction_data,
6593                                                          n_instructions);
6594
6595         return n_instructions;
6596 }
6597
6598 static int
6599 instruction_config(struct rte_swx_pipeline *p,
6600                    struct action *a,
6601                    const char **instructions,
6602                    uint32_t n_instructions)
6603 {
6604         struct instruction *instr = NULL;
6605         struct instruction_data *data = NULL;
6606         int err = 0;
6607         uint32_t i;
6608
6609         CHECK(n_instructions, EINVAL);
6610         CHECK(instructions, EINVAL);
6611         for (i = 0; i < n_instructions; i++)
6612                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6613
6614         /* Memory allocation. */
6615         instr = calloc(n_instructions, sizeof(struct instruction));
6616         if (!instr) {
6617                 err = -ENOMEM;
6618                 goto error;
6619         }
6620
6621         data = calloc(n_instructions, sizeof(struct instruction_data));
6622         if (!data) {
6623                 err = -ENOMEM;
6624                 goto error;
6625         }
6626
6627         for (i = 0; i < n_instructions; i++) {
6628                 char *string = strdup(instructions[i]);
6629                 if (!string) {
6630                         err = -ENOMEM;
6631                         goto error;
6632                 }
6633
6634                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6635                 if (err) {
6636                         free(string);
6637                         goto error;
6638                 }
6639
6640                 free(string);
6641         }
6642
6643         err = instr_label_check(data, n_instructions);
6644         if (err)
6645                 goto error;
6646
6647         err = instr_verify(p, a, instr, data, n_instructions);
6648         if (err)
6649                 goto error;
6650
6651         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6652
6653         err = instr_jmp_resolve(instr, data, n_instructions);
6654         if (err)
6655                 goto error;
6656
6657         if (a) {
6658                 a->instructions = instr;
6659                 a->instruction_data = data;
6660                 a->n_instructions = n_instructions;
6661         } else {
6662                 p->instructions = instr;
6663                 p->instruction_data = data;
6664                 p->n_instructions = n_instructions;
6665         }
6666
6667         return 0;
6668
6669 error:
6670         free(data);
6671         free(instr);
6672         return err;
6673 }
6674
6675 static instr_exec_t instruction_table[] = {
6676         [INSTR_RX] = instr_rx_exec,
6677         [INSTR_TX] = instr_tx_exec,
6678         [INSTR_TX_I] = instr_tx_i_exec,
6679         [INSTR_DROP] = instr_drop_exec,
6680
6681         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6682         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6683         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6684         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6685         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6686         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6687         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6688         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6689         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6690         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6691
6692         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6693         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6694         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6695         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6696         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6697         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6698         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6699         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6700         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6701
6702         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6703         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6704
6705         [INSTR_MOV] = instr_mov_exec,
6706         [INSTR_MOV_MH] = instr_mov_mh_exec,
6707         [INSTR_MOV_HM] = instr_mov_hm_exec,
6708         [INSTR_MOV_HH] = instr_mov_hh_exec,
6709         [INSTR_MOV_I] = instr_mov_i_exec,
6710
6711         [INSTR_DMA_HT] = instr_dma_ht_exec,
6712         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6713         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6714         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6715         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6716         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6717         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6718         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6719
6720         [INSTR_ALU_ADD] = instr_alu_add_exec,
6721         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6722         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6723         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6724         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6725         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6726
6727         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6728         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6729         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6730         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6731         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6732         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6733
6734         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6735         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6736         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6737         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6738
6739         [INSTR_ALU_AND] = instr_alu_and_exec,
6740         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6741         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6742         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6743         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6744
6745         [INSTR_ALU_OR] = instr_alu_or_exec,
6746         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6747         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6748         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6749         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6750
6751         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6752         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6753         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6754         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6755         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6756
6757         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6758         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6759         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6760         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6761         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6762         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6763
6764         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6765         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6766         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6767         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6768         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6769         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6770
6771         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6772         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6773         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6774
6775         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6776         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6777         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6778         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6779         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6780         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6781
6782         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6783         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6784         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6785         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6786         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6787         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6788         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6789         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6790         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6791
6792         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6793         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6794         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6795         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6796         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6797         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6798         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6799         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6800         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6801
6802         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6803         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6804         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6805
6806         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6807         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6808         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6809         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6810         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6811         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6812         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6813         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6814         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6815         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6816         [INSTR_METER_IMM] = instr_meter_imm_exec,
6817         [INSTR_METER_IMI] = instr_meter_imi_exec,
6818
6819         [INSTR_TABLE] = instr_table_exec,
6820         [INSTR_TABLE_AF] = instr_table_af_exec,
6821         [INSTR_SELECTOR] = instr_selector_exec,
6822         [INSTR_LEARNER] = instr_learner_exec,
6823         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6824         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6825         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6826         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6827         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6828
6829         [INSTR_JMP] = instr_jmp_exec,
6830         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6831         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6832         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6833         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6834         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6835         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6836
6837         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6838         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6839         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6840         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6841         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6842
6843         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6844         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6845         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6846         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6847         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6848
6849         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6850         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6851         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6852         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6853         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6854         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6855
6856         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6857         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6858         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6859         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6860         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6861         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6862
6863         [INSTR_RETURN] = instr_return_exec,
6864 };
6865
6866 static int
6867 instruction_table_build(struct rte_swx_pipeline *p)
6868 {
6869         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6870                                       sizeof(struct instr_exec_t *));
6871         if (!p->instruction_table)
6872                 return -EINVAL;
6873
6874         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6875
6876         return 0;
6877 }
6878
6879 static void
6880 instruction_table_build_free(struct rte_swx_pipeline *p)
6881 {
6882         if (!p->instruction_table)
6883                 return;
6884
6885         free(p->instruction_table);
6886         p->instruction_table = NULL;
6887 }
6888
6889 static void
6890 instruction_table_free(struct rte_swx_pipeline *p)
6891 {
6892         instruction_table_build_free(p);
6893 }
6894
6895 static inline void
6896 instr_exec(struct rte_swx_pipeline *p)
6897 {
6898         struct thread *t = &p->threads[p->thread_id];
6899         struct instruction *ip = t->ip;
6900         instr_exec_t instr = p->instruction_table[ip->type];
6901
6902         instr(p);
6903 }
6904
6905 /*
6906  * Action.
6907  */
6908 static struct action *
6909 action_find(struct rte_swx_pipeline *p, const char *name)
6910 {
6911         struct action *elem;
6912
6913         if (!name)
6914                 return NULL;
6915
6916         TAILQ_FOREACH(elem, &p->actions, node)
6917                 if (strcmp(elem->name, name) == 0)
6918                         return elem;
6919
6920         return NULL;
6921 }
6922
6923 static struct action *
6924 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6925 {
6926         struct action *action = NULL;
6927
6928         TAILQ_FOREACH(action, &p->actions, node)
6929                 if (action->id == id)
6930                         return action;
6931
6932         return NULL;
6933 }
6934
6935 static struct field *
6936 action_field_find(struct action *a, const char *name)
6937 {
6938         return a->st ? struct_type_field_find(a->st, name) : NULL;
6939 }
6940
6941 static struct field *
6942 action_field_parse(struct action *action, const char *name)
6943 {
6944         if (name[0] != 't' || name[1] != '.')
6945                 return NULL;
6946
6947         return action_field_find(action, &name[2]);
6948 }
6949
6950 static int
6951 action_has_nbo_args(struct action *a)
6952 {
6953         uint32_t i;
6954
6955         /* Return if the action does not have any args. */
6956         if (!a->st)
6957                 return 0; /* FALSE */
6958
6959         for (i = 0; i < a->st->n_fields; i++)
6960                 if (a->args_endianness[i])
6961                         return 1; /* TRUE */
6962
6963         return 0; /* FALSE */
6964 }
6965
6966 static int
6967 action_does_learning(struct action *a)
6968 {
6969         uint32_t i;
6970
6971         for (i = 0; i < a->n_instructions; i++)
6972                 switch (a->instructions[i].type) {
6973                 case INSTR_LEARNER_LEARN:
6974                         return 1; /* TRUE */
6975
6976                 case INSTR_LEARNER_FORGET:
6977                         return 1; /* TRUE */
6978
6979                 default:
6980                         continue;
6981                 }
6982
6983         return 0; /* FALSE */
6984 }
6985
6986 int
6987 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6988                                const char *name,
6989                                const char *args_struct_type_name,
6990                                const char **instructions,
6991                                uint32_t n_instructions)
6992 {
6993         struct struct_type *args_struct_type = NULL;
6994         struct action *a;
6995         int err;
6996
6997         CHECK(p, EINVAL);
6998
6999         CHECK_NAME(name, EINVAL);
7000         CHECK(!action_find(p, name), EEXIST);
7001
7002         if (args_struct_type_name) {
7003                 CHECK_NAME(args_struct_type_name, EINVAL);
7004                 args_struct_type = struct_type_find(p, args_struct_type_name);
7005                 CHECK(args_struct_type, EINVAL);
7006                 CHECK(!args_struct_type->var_size, EINVAL);
7007         }
7008
7009         /* Node allocation. */
7010         a = calloc(1, sizeof(struct action));
7011         CHECK(a, ENOMEM);
7012         if (args_struct_type) {
7013                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7014                 if (!a->args_endianness) {
7015                         free(a);
7016                         CHECK(0, ENOMEM);
7017                 }
7018         }
7019
7020         /* Node initialization. */
7021         strcpy(a->name, name);
7022         a->st = args_struct_type;
7023         a->id = p->n_actions;
7024
7025         /* Instruction translation. */
7026         err = instruction_config(p, a, instructions, n_instructions);
7027         if (err) {
7028                 free(a->args_endianness);
7029                 free(a);
7030                 return err;
7031         }
7032
7033         /* Node add to tailq. */
7034         TAILQ_INSERT_TAIL(&p->actions, a, node);
7035         p->n_actions++;
7036
7037         return 0;
7038 }
7039
7040 static int
7041 action_build(struct rte_swx_pipeline *p)
7042 {
7043         struct action *action;
7044
7045         /* p->action_instructions. */
7046         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7047         CHECK(p->action_instructions, ENOMEM);
7048
7049         TAILQ_FOREACH(action, &p->actions, node)
7050                 p->action_instructions[action->id] = action->instructions;
7051
7052         /* p->action_funcs. */
7053         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7054         CHECK(p->action_funcs, ENOMEM);
7055
7056         return 0;
7057 }
7058
7059 static void
7060 action_build_free(struct rte_swx_pipeline *p)
7061 {
7062         free(p->action_funcs);
7063         p->action_funcs = NULL;
7064
7065         free(p->action_instructions);
7066         p->action_instructions = NULL;
7067 }
7068
7069 static void
7070 action_free(struct rte_swx_pipeline *p)
7071 {
7072         action_build_free(p);
7073
7074         for ( ; ; ) {
7075                 struct action *action;
7076
7077                 action = TAILQ_FIRST(&p->actions);
7078                 if (!action)
7079                         break;
7080
7081                 TAILQ_REMOVE(&p->actions, action, node);
7082                 free(action->instruction_data);
7083                 free(action->instructions);
7084                 free(action);
7085         }
7086 }
7087
7088 static uint32_t
7089 action_arg_src_mov_count(struct action *a,
7090                          uint32_t arg_id,
7091                          struct instruction *instructions,
7092                          struct instruction_data *instruction_data,
7093                          uint32_t n_instructions)
7094 {
7095         uint32_t offset, n_users = 0, i;
7096
7097         if (!a->st ||
7098             (arg_id >= a->st->n_fields) ||
7099             !instructions ||
7100             !instruction_data ||
7101             !n_instructions)
7102                 return 0;
7103
7104         offset = a->st->fields[arg_id].offset / 8;
7105
7106         for (i = 0; i < n_instructions; i++) {
7107                 struct instruction *instr = &instructions[i];
7108                 struct instruction_data *data = &instruction_data[i];
7109
7110                 if (data->invalid ||
7111                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7112                     instr->mov.src.struct_id ||
7113                     (instr->mov.src.offset != offset))
7114                         continue;
7115
7116                 n_users++;
7117         }
7118
7119         return n_users;
7120 }
7121
7122 /*
7123  * Table.
7124  */
7125 static struct table_type *
7126 table_type_find(struct rte_swx_pipeline *p, const char *name)
7127 {
7128         struct table_type *elem;
7129
7130         TAILQ_FOREACH(elem, &p->table_types, node)
7131                 if (strcmp(elem->name, name) == 0)
7132                         return elem;
7133
7134         return NULL;
7135 }
7136
7137 static struct table_type *
7138 table_type_resolve(struct rte_swx_pipeline *p,
7139                    const char *recommended_type_name,
7140                    enum rte_swx_table_match_type match_type)
7141 {
7142         struct table_type *elem;
7143
7144         /* Only consider the recommended type if the match type is correct. */
7145         if (recommended_type_name)
7146                 TAILQ_FOREACH(elem, &p->table_types, node)
7147                         if (!strcmp(elem->name, recommended_type_name) &&
7148                             (elem->match_type == match_type))
7149                                 return elem;
7150
7151         /* Ignore the recommended type and get the first element with this match
7152          * type.
7153          */
7154         TAILQ_FOREACH(elem, &p->table_types, node)
7155                 if (elem->match_type == match_type)
7156                         return elem;
7157
7158         return NULL;
7159 }
7160
7161 static struct table *
7162 table_find(struct rte_swx_pipeline *p, const char *name)
7163 {
7164         struct table *elem;
7165
7166         TAILQ_FOREACH(elem, &p->tables, node)
7167                 if (strcmp(elem->name, name) == 0)
7168                         return elem;
7169
7170         return NULL;
7171 }
7172
7173 static struct table *
7174 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7175 {
7176         struct table *table = NULL;
7177
7178         TAILQ_FOREACH(table, &p->tables, node)
7179                 if (table->id == id)
7180                         return table;
7181
7182         return NULL;
7183 }
7184
7185 int
7186 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7187                                      const char *name,
7188                                      enum rte_swx_table_match_type match_type,
7189                                      struct rte_swx_table_ops *ops)
7190 {
7191         struct table_type *elem;
7192
7193         CHECK(p, EINVAL);
7194
7195         CHECK_NAME(name, EINVAL);
7196         CHECK(!table_type_find(p, name), EEXIST);
7197
7198         CHECK(ops, EINVAL);
7199         CHECK(ops->create, EINVAL);
7200         CHECK(ops->lkp, EINVAL);
7201         CHECK(ops->free, EINVAL);
7202
7203         /* Node allocation. */
7204         elem = calloc(1, sizeof(struct table_type));
7205         CHECK(elem, ENOMEM);
7206
7207         /* Node initialization. */
7208         strcpy(elem->name, name);
7209         elem->match_type = match_type;
7210         memcpy(&elem->ops, ops, sizeof(*ops));
7211
7212         /* Node add to tailq. */
7213         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7214
7215         return 0;
7216 }
7217
7218 static int
7219 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7220                          uint32_t n_fields,
7221                          enum rte_swx_table_match_type *match_type)
7222 {
7223         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7224
7225         for (i = 0; i < n_fields; i++) {
7226                 struct rte_swx_match_field_params  *f = &fields[i];
7227
7228                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7229                         n_fields_em++;
7230
7231                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7232                         n_fields_lpm++;
7233         }
7234
7235         if ((n_fields_lpm > 1) ||
7236             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7237                 return -EINVAL;
7238
7239         *match_type = (n_fields_em == n_fields) ?
7240                        RTE_SWX_TABLE_MATCH_EXACT :
7241                        RTE_SWX_TABLE_MATCH_WILDCARD;
7242
7243         return 0;
7244 }
7245
7246 static int
7247 table_match_fields_check(struct rte_swx_pipeline *p,
7248                          struct rte_swx_pipeline_table_params *params,
7249                          struct header **header)
7250 {
7251         struct header *h0 = NULL;
7252         struct field *hf, *mf;
7253         uint32_t *offset = NULL, i;
7254         int status = 0;
7255
7256         /* Return if no match fields. */
7257         if (!params->n_fields) {
7258                 if (params->fields) {
7259                         status = -EINVAL;
7260                         goto end;
7261                 }
7262
7263                 if (header)
7264                         *header = NULL;
7265
7266                 return 0;
7267         }
7268
7269         /* Memory allocation. */
7270         offset = calloc(params->n_fields, sizeof(uint32_t));
7271         if (!offset) {
7272                 status = -ENOMEM;
7273                 goto end;
7274         }
7275
7276         /* Check that all the match fields belong to either the same header or
7277          * to the meta-data.
7278          */
7279         hf = header_field_parse(p, params->fields[0].name, &h0);
7280         mf = metadata_field_parse(p, params->fields[0].name);
7281         if ((!hf && !mf) || (hf && hf->var_size)) {
7282                 status = -EINVAL;
7283                 goto end;
7284         }
7285
7286         offset[0] = h0 ? hf->offset : mf->offset;
7287
7288         for (i = 1; i < params->n_fields; i++)
7289                 if (h0) {
7290                         struct header *h;
7291
7292                         hf = header_field_parse(p, params->fields[i].name, &h);
7293                         if (!hf || (h->id != h0->id) || hf->var_size) {
7294                                 status = -EINVAL;
7295                                 goto end;
7296                         }
7297
7298                         offset[i] = hf->offset;
7299                 } else {
7300                         mf = metadata_field_parse(p, params->fields[i].name);
7301                         if (!mf) {
7302                                 status = -EINVAL;
7303                                 goto end;
7304                         }
7305
7306                         offset[i] = mf->offset;
7307                 }
7308
7309         /* Check that there are no duplicated match fields. */
7310         for (i = 0; i < params->n_fields; i++) {
7311                 uint32_t j;
7312
7313                 for (j = 0; j < i; j++)
7314                         if (offset[j] == offset[i]) {
7315                                 status = -EINVAL;
7316                                 goto end;
7317                         }
7318         }
7319
7320         /* Return. */
7321         if (header)
7322                 *header = h0;
7323
7324 end:
7325         free(offset);
7326         return status;
7327 }
7328
7329 int
7330 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7331                               const char *name,
7332                               struct rte_swx_pipeline_table_params *params,
7333                               const char *recommended_table_type_name,
7334                               const char *args,
7335                               uint32_t size)
7336 {
7337         struct table_type *type;
7338         struct table *t = NULL;
7339         struct action *default_action;
7340         struct header *header = NULL;
7341         uint32_t action_data_size_max = 0, i;
7342         int status = 0;
7343
7344         CHECK(p, EINVAL);
7345
7346         CHECK_NAME(name, EINVAL);
7347         CHECK(!table_find(p, name), EEXIST);
7348         CHECK(!selector_find(p, name), EEXIST);
7349         CHECK(!learner_find(p, name), EEXIST);
7350
7351         CHECK(params, EINVAL);
7352
7353         /* Match checks. */
7354         status = table_match_fields_check(p, params, &header);
7355         if (status)
7356                 return status;
7357
7358         /* Action checks. */
7359         CHECK(params->n_actions, EINVAL);
7360         CHECK(params->action_names, EINVAL);
7361         for (i = 0; i < params->n_actions; i++) {
7362                 const char *action_name = params->action_names[i];
7363                 struct action *a;
7364                 uint32_t action_data_size;
7365                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7366
7367                 CHECK_NAME(action_name, EINVAL);
7368
7369                 a = action_find(p, action_name);
7370                 CHECK(a, EINVAL);
7371                 CHECK(!action_does_learning(a), EINVAL);
7372
7373                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7374                 if (action_data_size > action_data_size_max)
7375                         action_data_size_max = action_data_size;
7376
7377                 if (params->action_is_for_table_entries)
7378                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7379                 if (params->action_is_for_default_entry)
7380                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7381                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7382         }
7383
7384         CHECK_NAME(params->default_action_name, EINVAL);
7385         for (i = 0; i < p->n_actions; i++)
7386                 if (!strcmp(params->action_names[i],
7387                             params->default_action_name))
7388                         break;
7389         CHECK(i < params->n_actions, EINVAL);
7390         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7391               EINVAL);
7392
7393         default_action = action_find(p, params->default_action_name);
7394         CHECK((default_action->st && params->default_action_data) ||
7395               !params->default_action_data, EINVAL);
7396
7397         /* Table type checks. */
7398         if (recommended_table_type_name)
7399                 CHECK_NAME(recommended_table_type_name, EINVAL);
7400
7401         if (params->n_fields) {
7402                 enum rte_swx_table_match_type match_type;
7403
7404                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7405                 if (status)
7406                         return status;
7407
7408                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7409                 CHECK(type, EINVAL);
7410         } else {
7411                 type = NULL;
7412         }
7413
7414         /* Memory allocation. */
7415         t = calloc(1, sizeof(struct table));
7416         if (!t)
7417                 goto nomem;
7418
7419         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7420         if (!t->fields)
7421                 goto nomem;
7422
7423         t->actions = calloc(params->n_actions, sizeof(struct action *));
7424         if (!t->actions)
7425                 goto nomem;
7426
7427         if (action_data_size_max) {
7428                 t->default_action_data = calloc(1, action_data_size_max);
7429                 if (!t->default_action_data)
7430                         goto nomem;
7431         }
7432
7433         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7434         if (!t->action_is_for_table_entries)
7435                 goto nomem;
7436
7437         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7438         if (!t->action_is_for_default_entry)
7439                 goto nomem;
7440
7441         /* Node initialization. */
7442         strcpy(t->name, name);
7443         if (args && args[0])
7444                 strcpy(t->args, args);
7445         t->type = type;
7446
7447         for (i = 0; i < params->n_fields; i++) {
7448                 struct rte_swx_match_field_params *field = &params->fields[i];
7449                 struct match_field *f = &t->fields[i];
7450
7451                 f->match_type = field->match_type;
7452                 f->field = header ?
7453                         header_field_parse(p, field->name, NULL) :
7454                         metadata_field_parse(p, field->name);
7455         }
7456         t->n_fields = params->n_fields;
7457         t->header = header;
7458
7459         for (i = 0; i < params->n_actions; i++) {
7460                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7461
7462                 if (params->action_is_for_table_entries)
7463                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7464                 if (params->action_is_for_default_entry)
7465                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7466
7467                 t->actions[i] = action_find(p, params->action_names[i]);
7468                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7469                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7470         }
7471         t->default_action = default_action;
7472         if (default_action->st)
7473                 memcpy(t->default_action_data,
7474                        params->default_action_data,
7475                        default_action->st->n_bits / 8);
7476         t->n_actions = params->n_actions;
7477         t->default_action_is_const = params->default_action_is_const;
7478         t->action_data_size_max = action_data_size_max;
7479
7480         t->size = size;
7481         t->id = p->n_tables;
7482
7483         /* Node add to tailq. */
7484         TAILQ_INSERT_TAIL(&p->tables, t, node);
7485         p->n_tables++;
7486
7487         return 0;
7488
7489 nomem:
7490         if (!t)
7491                 return -ENOMEM;
7492
7493         free(t->action_is_for_default_entry);
7494         free(t->action_is_for_table_entries);
7495         free(t->default_action_data);
7496         free(t->actions);
7497         free(t->fields);
7498         free(t);
7499
7500         return -ENOMEM;
7501 }
7502
7503 static struct rte_swx_table_params *
7504 table_params_get(struct table *table)
7505 {
7506         struct rte_swx_table_params *params;
7507         struct field *first, *last;
7508         uint8_t *key_mask;
7509         uint32_t key_size, key_offset, action_data_size, i;
7510
7511         /* Memory allocation. */
7512         params = calloc(1, sizeof(struct rte_swx_table_params));
7513         if (!params)
7514                 return NULL;
7515
7516         /* Find first (smallest offset) and last (biggest offset) match fields. */
7517         first = table->fields[0].field;
7518         last = table->fields[0].field;
7519
7520         for (i = 0; i < table->n_fields; i++) {
7521                 struct field *f = table->fields[i].field;
7522
7523                 if (f->offset < first->offset)
7524                         first = f;
7525
7526                 if (f->offset > last->offset)
7527                         last = f;
7528         }
7529
7530         /* Key offset and size. */
7531         key_offset = first->offset / 8;
7532         key_size = (last->offset + last->n_bits - first->offset) / 8;
7533
7534         /* Memory allocation. */
7535         key_mask = calloc(1, key_size);
7536         if (!key_mask) {
7537                 free(params);
7538                 return NULL;
7539         }
7540
7541         /* Key mask. */
7542         for (i = 0; i < table->n_fields; i++) {
7543                 struct field *f = table->fields[i].field;
7544                 uint32_t start = (f->offset - first->offset) / 8;
7545                 size_t size = f->n_bits / 8;
7546
7547                 memset(&key_mask[start], 0xFF, size);
7548         }
7549
7550         /* Action data size. */
7551         action_data_size = 0;
7552         for (i = 0; i < table->n_actions; i++) {
7553                 struct action *action = table->actions[i];
7554                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7555
7556                 if (ads > action_data_size)
7557                         action_data_size = ads;
7558         }
7559
7560         /* Fill in. */
7561         params->match_type = table->type->match_type;
7562         params->key_size = key_size;
7563         params->key_offset = key_offset;
7564         params->key_mask0 = key_mask;
7565         params->action_data_size = action_data_size;
7566         params->n_keys_max = table->size;
7567
7568         return params;
7569 }
7570
7571 static void
7572 table_params_free(struct rte_swx_table_params *params)
7573 {
7574         if (!params)
7575                 return;
7576
7577         free(params->key_mask0);
7578         free(params);
7579 }
7580
7581 static int
7582 table_stub_lkp(void *table __rte_unused,
7583                void *mailbox __rte_unused,
7584                uint8_t **key __rte_unused,
7585                uint64_t *action_id __rte_unused,
7586                uint8_t **action_data __rte_unused,
7587                int *hit)
7588 {
7589         *hit = 0;
7590         return 1; /* DONE. */
7591 }
7592
7593 static int
7594 table_build(struct rte_swx_pipeline *p)
7595 {
7596         uint32_t i;
7597
7598         /* Per pipeline: table statistics. */
7599         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7600         CHECK(p->table_stats, ENOMEM);
7601
7602         for (i = 0; i < p->n_tables; i++) {
7603                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7604                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7605         }
7606
7607         /* Per thread: table runt-time. */
7608         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7609                 struct thread *t = &p->threads[i];
7610                 struct table *table;
7611
7612                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7613                 CHECK(t->tables, ENOMEM);
7614
7615                 TAILQ_FOREACH(table, &p->tables, node) {
7616                         struct table_runtime *r = &t->tables[table->id];
7617
7618                         if (table->type) {
7619                                 uint64_t size;
7620
7621                                 size = table->type->ops.mailbox_size_get();
7622
7623                                 /* r->func. */
7624                                 r->func = table->type->ops.lkp;
7625
7626                                 /* r->mailbox. */
7627                                 if (size) {
7628                                         r->mailbox = calloc(1, size);
7629                                         CHECK(r->mailbox, ENOMEM);
7630                                 }
7631
7632                                 /* r->key. */
7633                                 r->key = table->header ?
7634                                         &t->structs[table->header->struct_id] :
7635                                         &t->structs[p->metadata_struct_id];
7636                         } else {
7637                                 r->func = table_stub_lkp;
7638                         }
7639                 }
7640         }
7641
7642         return 0;
7643 }
7644
7645 static void
7646 table_build_free(struct rte_swx_pipeline *p)
7647 {
7648         uint32_t i;
7649
7650         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7651                 struct thread *t = &p->threads[i];
7652                 uint32_t j;
7653
7654                 if (!t->tables)
7655                         continue;
7656
7657                 for (j = 0; j < p->n_tables; j++) {
7658                         struct table_runtime *r = &t->tables[j];
7659
7660                         free(r->mailbox);
7661                 }
7662
7663                 free(t->tables);
7664                 t->tables = NULL;
7665         }
7666
7667         if (p->table_stats) {
7668                 for (i = 0; i < p->n_tables; i++)
7669                         free(p->table_stats[i].n_pkts_action);
7670
7671                 free(p->table_stats);
7672         }
7673 }
7674
7675 static void
7676 table_free(struct rte_swx_pipeline *p)
7677 {
7678         table_build_free(p);
7679
7680         /* Tables. */
7681         for ( ; ; ) {
7682                 struct table *elem;
7683
7684                 elem = TAILQ_FIRST(&p->tables);
7685                 if (!elem)
7686                         break;
7687
7688                 TAILQ_REMOVE(&p->tables, elem, node);
7689                 free(elem->fields);
7690                 free(elem->actions);
7691                 free(elem->default_action_data);
7692                 free(elem);
7693         }
7694
7695         /* Table types. */
7696         for ( ; ; ) {
7697                 struct table_type *elem;
7698
7699                 elem = TAILQ_FIRST(&p->table_types);
7700                 if (!elem)
7701                         break;
7702
7703                 TAILQ_REMOVE(&p->table_types, elem, node);
7704                 free(elem);
7705         }
7706 }
7707
7708 /*
7709  * Selector.
7710  */
7711 static struct selector *
7712 selector_find(struct rte_swx_pipeline *p, const char *name)
7713 {
7714         struct selector *s;
7715
7716         TAILQ_FOREACH(s, &p->selectors, node)
7717                 if (strcmp(s->name, name) == 0)
7718                         return s;
7719
7720         return NULL;
7721 }
7722
7723 static struct selector *
7724 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7725 {
7726         struct selector *s = NULL;
7727
7728         TAILQ_FOREACH(s, &p->selectors, node)
7729                 if (s->id == id)
7730                         return s;
7731
7732         return NULL;
7733 }
7734
7735 static int
7736 selector_fields_check(struct rte_swx_pipeline *p,
7737                       struct rte_swx_pipeline_selector_params *params,
7738                       struct header **header)
7739 {
7740         struct header *h0 = NULL;
7741         struct field *hf, *mf;
7742         uint32_t i;
7743
7744         /* Return if no selector fields. */
7745         if (!params->n_selector_fields || !params->selector_field_names)
7746                 return -EINVAL;
7747
7748         /* Check that all the selector fields either belong to the same header
7749          * or are all meta-data fields.
7750          */
7751         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7752         mf = metadata_field_parse(p, params->selector_field_names[0]);
7753         if (!hf && !mf)
7754                 return -EINVAL;
7755
7756         for (i = 1; i < params->n_selector_fields; i++)
7757                 if (h0) {
7758                         struct header *h;
7759
7760                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7761                         if (!hf || (h->id != h0->id))
7762                                 return -EINVAL;
7763                 } else {
7764                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7765                         if (!mf)
7766                                 return -EINVAL;
7767                 }
7768
7769         /* Check that there are no duplicated match fields. */
7770         for (i = 0; i < params->n_selector_fields; i++) {
7771                 const char *field_name = params->selector_field_names[i];
7772                 uint32_t j;
7773
7774                 for (j = i + 1; j < params->n_selector_fields; j++)
7775                         if (!strcmp(params->selector_field_names[j], field_name))
7776                                 return -EINVAL;
7777         }
7778
7779         /* Return. */
7780         if (header)
7781                 *header = h0;
7782
7783         return 0;
7784 }
7785
7786 int
7787 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7788                                  const char *name,
7789                                  struct rte_swx_pipeline_selector_params *params)
7790 {
7791         struct selector *s;
7792         struct header *selector_header = NULL;
7793         struct field *group_id_field, *member_id_field;
7794         uint32_t i;
7795         int status = 0;
7796
7797         CHECK(p, EINVAL);
7798
7799         CHECK_NAME(name, EINVAL);
7800         CHECK(!table_find(p, name), EEXIST);
7801         CHECK(!selector_find(p, name), EEXIST);
7802         CHECK(!learner_find(p, name), EEXIST);
7803
7804         CHECK(params, EINVAL);
7805
7806         CHECK_NAME(params->group_id_field_name, EINVAL);
7807         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7808         CHECK(group_id_field, EINVAL);
7809
7810         for (i = 0; i < params->n_selector_fields; i++) {
7811                 const char *field_name = params->selector_field_names[i];
7812
7813                 CHECK_NAME(field_name, EINVAL);
7814         }
7815         status = selector_fields_check(p, params, &selector_header);
7816         if (status)
7817                 return status;
7818
7819         CHECK_NAME(params->member_id_field_name, EINVAL);
7820         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7821         CHECK(member_id_field, EINVAL);
7822
7823         CHECK(params->n_groups_max, EINVAL);
7824
7825         CHECK(params->n_members_per_group_max, EINVAL);
7826
7827         /* Memory allocation. */
7828         s = calloc(1, sizeof(struct selector));
7829         if (!s) {
7830                 status = -ENOMEM;
7831                 goto error;
7832         }
7833
7834         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7835         if (!s->selector_fields) {
7836                 status = -ENOMEM;
7837                 goto error;
7838         }
7839
7840         /* Node initialization. */
7841         strcpy(s->name, name);
7842
7843         s->group_id_field = group_id_field;
7844
7845         for (i = 0; i < params->n_selector_fields; i++) {
7846                 const char *field_name = params->selector_field_names[i];
7847
7848                 s->selector_fields[i] = selector_header ?
7849                         header_field_parse(p, field_name, NULL) :
7850                         metadata_field_parse(p, field_name);
7851         }
7852
7853         s->n_selector_fields = params->n_selector_fields;
7854
7855         s->selector_header = selector_header;
7856
7857         s->member_id_field = member_id_field;
7858
7859         s->n_groups_max = params->n_groups_max;
7860
7861         s->n_members_per_group_max = params->n_members_per_group_max;
7862
7863         s->id = p->n_selectors;
7864
7865         /* Node add to tailq. */
7866         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7867         p->n_selectors++;
7868
7869         return 0;
7870
7871 error:
7872         if (!s)
7873                 return status;
7874
7875         free(s->selector_fields);
7876
7877         free(s);
7878
7879         return status;
7880 }
7881
7882 static void
7883 selector_params_free(struct rte_swx_table_selector_params *params)
7884 {
7885         if (!params)
7886                 return;
7887
7888         free(params->selector_mask);
7889
7890         free(params);
7891 }
7892
7893 static struct rte_swx_table_selector_params *
7894 selector_table_params_get(struct selector *s)
7895 {
7896         struct rte_swx_table_selector_params *params = NULL;
7897         struct field *first, *last;
7898         uint32_t i;
7899
7900         /* Memory allocation. */
7901         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7902         if (!params)
7903                 goto error;
7904
7905         /* Group ID. */
7906         params->group_id_offset = s->group_id_field->offset / 8;
7907
7908         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7909         first = s->selector_fields[0];
7910         last = s->selector_fields[0];
7911
7912         for (i = 0; i < s->n_selector_fields; i++) {
7913                 struct field *f = s->selector_fields[i];
7914
7915                 if (f->offset < first->offset)
7916                         first = f;
7917
7918                 if (f->offset > last->offset)
7919                         last = f;
7920         }
7921
7922         /* Selector offset and size. */
7923         params->selector_offset = first->offset / 8;
7924         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7925
7926         /* Memory allocation. */
7927         params->selector_mask = calloc(1, params->selector_size);
7928         if (!params->selector_mask)
7929                 goto error;
7930
7931         /* Selector mask. */
7932         for (i = 0; i < s->n_selector_fields; i++) {
7933                 struct field *f = s->selector_fields[i];
7934                 uint32_t start = (f->offset - first->offset) / 8;
7935                 size_t size = f->n_bits / 8;
7936
7937                 memset(&params->selector_mask[start], 0xFF, size);
7938         }
7939
7940         /* Member ID. */
7941         params->member_id_offset = s->member_id_field->offset / 8;
7942
7943         /* Maximum number of groups. */
7944         params->n_groups_max = s->n_groups_max;
7945
7946         /* Maximum number of members per group. */
7947         params->n_members_per_group_max = s->n_members_per_group_max;
7948
7949         return params;
7950
7951 error:
7952         selector_params_free(params);
7953         return NULL;
7954 }
7955
7956 static void
7957 selector_build_free(struct rte_swx_pipeline *p)
7958 {
7959         uint32_t i;
7960
7961         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7962                 struct thread *t = &p->threads[i];
7963                 uint32_t j;
7964
7965                 if (!t->selectors)
7966                         continue;
7967
7968                 for (j = 0; j < p->n_selectors; j++) {
7969                         struct selector_runtime *r = &t->selectors[j];
7970
7971                         free(r->mailbox);
7972                 }
7973
7974                 free(t->selectors);
7975                 t->selectors = NULL;
7976         }
7977
7978         free(p->selector_stats);
7979         p->selector_stats = NULL;
7980 }
7981
7982 static int
7983 selector_build(struct rte_swx_pipeline *p)
7984 {
7985         uint32_t i;
7986         int status = 0;
7987
7988         /* Per pipeline: selector statistics. */
7989         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7990         if (!p->selector_stats) {
7991                 status = -ENOMEM;
7992                 goto error;
7993         }
7994
7995         /* Per thread: selector run-time. */
7996         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7997                 struct thread *t = &p->threads[i];
7998                 struct selector *s;
7999
8000                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8001                 if (!t->selectors) {
8002                         status = -ENOMEM;
8003                         goto error;
8004                 }
8005
8006                 TAILQ_FOREACH(s, &p->selectors, node) {
8007                         struct selector_runtime *r = &t->selectors[s->id];
8008                         uint64_t size;
8009
8010                         /* r->mailbox. */
8011                         size = rte_swx_table_selector_mailbox_size_get();
8012                         if (size) {
8013                                 r->mailbox = calloc(1, size);
8014                                 if (!r->mailbox) {
8015                                         status = -ENOMEM;
8016                                         goto error;
8017                                 }
8018                         }
8019
8020                         /* r->group_id_buffer. */
8021                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8022
8023                         /* r->selector_buffer. */
8024                         r->selector_buffer = s->selector_header ?
8025                                 &t->structs[s->selector_header->struct_id] :
8026                                 &t->structs[p->metadata_struct_id];
8027
8028                         /* r->member_id_buffer. */
8029                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8030                 }
8031         }
8032
8033         return 0;
8034
8035 error:
8036         selector_build_free(p);
8037         return status;
8038 }
8039
8040 static void
8041 selector_free(struct rte_swx_pipeline *p)
8042 {
8043         selector_build_free(p);
8044
8045         /* Selector tables. */
8046         for ( ; ; ) {
8047                 struct selector *elem;
8048
8049                 elem = TAILQ_FIRST(&p->selectors);
8050                 if (!elem)
8051                         break;
8052
8053                 TAILQ_REMOVE(&p->selectors, elem, node);
8054                 free(elem->selector_fields);
8055                 free(elem);
8056         }
8057 }
8058
8059 /*
8060  * Learner table.
8061  */
8062 static struct learner *
8063 learner_find(struct rte_swx_pipeline *p, const char *name)
8064 {
8065         struct learner *l;
8066
8067         TAILQ_FOREACH(l, &p->learners, node)
8068                 if (!strcmp(l->name, name))
8069                         return l;
8070
8071         return NULL;
8072 }
8073
8074 static struct learner *
8075 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8076 {
8077         struct learner *l = NULL;
8078
8079         TAILQ_FOREACH(l, &p->learners, node)
8080                 if (l->id == id)
8081                         return l;
8082
8083         return NULL;
8084 }
8085
8086 static int
8087 learner_match_fields_check(struct rte_swx_pipeline *p,
8088                            struct rte_swx_pipeline_learner_params *params,
8089                            struct header **header)
8090 {
8091         struct header *h0 = NULL;
8092         struct field *hf, *mf;
8093         uint32_t i;
8094
8095         /* Return if no match fields. */
8096         if (!params->n_fields || !params->field_names)
8097                 return -EINVAL;
8098
8099         /* Check that all the match fields either belong to the same header
8100          * or are all meta-data fields.
8101          */
8102         hf = header_field_parse(p, params->field_names[0], &h0);
8103         mf = metadata_field_parse(p, params->field_names[0]);
8104         if (!hf && !mf)
8105                 return -EINVAL;
8106
8107         for (i = 1; i < params->n_fields; i++)
8108                 if (h0) {
8109                         struct header *h;
8110
8111                         hf = header_field_parse(p, params->field_names[i], &h);
8112                         if (!hf || (h->id != h0->id))
8113                                 return -EINVAL;
8114                 } else {
8115                         mf = metadata_field_parse(p, params->field_names[i]);
8116                         if (!mf)
8117                                 return -EINVAL;
8118                 }
8119
8120         /* Check that there are no duplicated match fields. */
8121         for (i = 0; i < params->n_fields; i++) {
8122                 const char *field_name = params->field_names[i];
8123                 uint32_t j;
8124
8125                 for (j = i + 1; j < params->n_fields; j++)
8126                         if (!strcmp(params->field_names[j], field_name))
8127                                 return -EINVAL;
8128         }
8129
8130         /* Return. */
8131         if (header)
8132                 *header = h0;
8133
8134         return 0;
8135 }
8136
8137 static int
8138 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8139 {
8140         struct struct_type *mst = p->metadata_st, *ast = a->st;
8141         struct field *mf, *af;
8142         uint32_t mf_pos, i;
8143
8144         if (!ast) {
8145                 if (mf_name)
8146                         return -EINVAL;
8147
8148                 return 0;
8149         }
8150
8151         /* Check that mf_name is the name of a valid meta-data field. */
8152         CHECK_NAME(mf_name, EINVAL);
8153         mf = metadata_field_parse(p, mf_name);
8154         CHECK(mf, EINVAL);
8155
8156         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8157          * all the action arguments.
8158          */
8159         mf_pos = mf - mst->fields;
8160         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8161
8162         /* Check that the size of each of the identified meta-data fields matches exactly the size
8163          * of the corresponding action argument.
8164          */
8165         for (i = 0; i < ast->n_fields; i++) {
8166                 mf = &mst->fields[mf_pos + i];
8167                 af = &ast->fields[i];
8168
8169                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8170         }
8171
8172         return 0;
8173 }
8174
8175 static int
8176 learner_action_learning_check(struct rte_swx_pipeline *p,
8177                               struct action *action,
8178                               const char **action_names,
8179                               uint32_t n_actions)
8180 {
8181         uint32_t i;
8182
8183         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8184          * the action passed as argument to the "learn" instruction) is also enabled for the
8185          * current learner table.
8186          */
8187         for (i = 0; i < action->n_instructions; i++) {
8188                 struct instruction *instr = &action->instructions[i];
8189                 uint32_t found = 0, j;
8190
8191                 if (instr->type != INSTR_LEARNER_LEARN)
8192                         continue;
8193
8194                 for (j = 0; j < n_actions; j++) {
8195                         struct action *a;
8196
8197                         a = action_find(p, action_names[j]);
8198                         if (!a)
8199                                 return -EINVAL;
8200
8201                         if (a->id == instr->learn.action_id)
8202                                 found = 1;
8203                 }
8204
8205                 if (!found)
8206                         return -EINVAL;
8207         }
8208
8209         return 0;
8210 }
8211
8212 int
8213 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8214                               const char *name,
8215                               struct rte_swx_pipeline_learner_params *params,
8216                               uint32_t size,
8217                               uint32_t timeout)
8218 {
8219         struct learner *l = NULL;
8220         struct action *default_action;
8221         struct header *header = NULL;
8222         uint32_t action_data_size_max = 0, i;
8223         int status = 0;
8224
8225         CHECK(p, EINVAL);
8226
8227         CHECK_NAME(name, EINVAL);
8228         CHECK(!table_find(p, name), EEXIST);
8229         CHECK(!selector_find(p, name), EEXIST);
8230         CHECK(!learner_find(p, name), EEXIST);
8231
8232         CHECK(params, EINVAL);
8233
8234         /* Match checks. */
8235         status = learner_match_fields_check(p, params, &header);
8236         if (status)
8237                 return status;
8238
8239         /* Action checks. */
8240         CHECK(params->n_actions, EINVAL);
8241         CHECK(params->action_names, EINVAL);
8242         for (i = 0; i < params->n_actions; i++) {
8243                 const char *action_name = params->action_names[i];
8244                 struct action *a;
8245                 uint32_t action_data_size;
8246                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8247
8248                 CHECK_NAME(action_name, EINVAL);
8249
8250                 a = action_find(p, action_name);
8251                 CHECK(a, EINVAL);
8252
8253                 status = learner_action_learning_check(p,
8254                                                        a,
8255                                                        params->action_names,
8256                                                        params->n_actions);
8257                 if (status)
8258                         return status;
8259
8260                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8261                 if (action_data_size > action_data_size_max)
8262                         action_data_size_max = action_data_size;
8263
8264                 if (params->action_is_for_table_entries)
8265                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8266                 if (params->action_is_for_default_entry)
8267                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8268                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8269         }
8270
8271         CHECK_NAME(params->default_action_name, EINVAL);
8272         for (i = 0; i < p->n_actions; i++)
8273                 if (!strcmp(params->action_names[i],
8274                             params->default_action_name))
8275                         break;
8276         CHECK(i < params->n_actions, EINVAL);
8277         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8278               EINVAL);
8279
8280         default_action = action_find(p, params->default_action_name);
8281         CHECK((default_action->st && params->default_action_data) ||
8282               !params->default_action_data, EINVAL);
8283
8284         /* Any other checks. */
8285         CHECK(size, EINVAL);
8286         CHECK(timeout, EINVAL);
8287
8288         /* Memory allocation. */
8289         l = calloc(1, sizeof(struct learner));
8290         if (!l)
8291                 goto nomem;
8292
8293         l->fields = calloc(params->n_fields, sizeof(struct field *));
8294         if (!l->fields)
8295                 goto nomem;
8296
8297         l->actions = calloc(params->n_actions, sizeof(struct action *));
8298         if (!l->actions)
8299                 goto nomem;
8300
8301         if (action_data_size_max) {
8302                 l->default_action_data = calloc(1, action_data_size_max);
8303                 if (!l->default_action_data)
8304                         goto nomem;
8305         }
8306
8307         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8308         if (!l->action_is_for_table_entries)
8309                 goto nomem;
8310
8311         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8312         if (!l->action_is_for_default_entry)
8313                 goto nomem;
8314
8315         /* Node initialization. */
8316         strcpy(l->name, name);
8317
8318         for (i = 0; i < params->n_fields; i++) {
8319                 const char *field_name = params->field_names[i];
8320
8321                 l->fields[i] = header ?
8322                         header_field_parse(p, field_name, NULL) :
8323                         metadata_field_parse(p, field_name);
8324         }
8325
8326         l->n_fields = params->n_fields;
8327
8328         l->header = header;
8329
8330         for (i = 0; i < params->n_actions; i++) {
8331                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8332
8333                 if (params->action_is_for_table_entries)
8334                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8335                 if (params->action_is_for_default_entry)
8336                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8337
8338                 l->actions[i] = action_find(p, params->action_names[i]);
8339                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8340                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8341         }
8342
8343         l->default_action = default_action;
8344
8345         if (default_action->st)
8346                 memcpy(l->default_action_data,
8347                        params->default_action_data,
8348                        default_action->st->n_bits / 8);
8349
8350         l->n_actions = params->n_actions;
8351
8352         l->default_action_is_const = params->default_action_is_const;
8353
8354         l->action_data_size_max = action_data_size_max;
8355
8356         l->size = size;
8357
8358         l->timeout = timeout;
8359
8360         l->id = p->n_learners;
8361
8362         /* Node add to tailq. */
8363         TAILQ_INSERT_TAIL(&p->learners, l, node);
8364         p->n_learners++;
8365
8366         return 0;
8367
8368 nomem:
8369         if (!l)
8370                 return -ENOMEM;
8371
8372         free(l->action_is_for_default_entry);
8373         free(l->action_is_for_table_entries);
8374         free(l->default_action_data);
8375         free(l->actions);
8376         free(l->fields);
8377         free(l);
8378
8379         return -ENOMEM;
8380 }
8381
8382 static void
8383 learner_params_free(struct rte_swx_table_learner_params *params)
8384 {
8385         if (!params)
8386                 return;
8387
8388         free(params->key_mask0);
8389
8390         free(params);
8391 }
8392
8393 static struct rte_swx_table_learner_params *
8394 learner_params_get(struct learner *l)
8395 {
8396         struct rte_swx_table_learner_params *params = NULL;
8397         struct field *first, *last;
8398         uint32_t i;
8399
8400         /* Memory allocation. */
8401         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8402         if (!params)
8403                 goto error;
8404
8405         /* Find first (smallest offset) and last (biggest offset) match fields. */
8406         first = l->fields[0];
8407         last = l->fields[0];
8408
8409         for (i = 0; i < l->n_fields; i++) {
8410                 struct field *f = l->fields[i];
8411
8412                 if (f->offset < first->offset)
8413                         first = f;
8414
8415                 if (f->offset > last->offset)
8416                         last = f;
8417         }
8418
8419         /* Key offset and size. */
8420         params->key_offset = first->offset / 8;
8421         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8422
8423         /* Memory allocation. */
8424         params->key_mask0 = calloc(1, params->key_size);
8425         if (!params->key_mask0)
8426                 goto error;
8427
8428         /* Key mask. */
8429         for (i = 0; i < l->n_fields; i++) {
8430                 struct field *f = l->fields[i];
8431                 uint32_t start = (f->offset - first->offset) / 8;
8432                 size_t size = f->n_bits / 8;
8433
8434                 memset(&params->key_mask0[start], 0xFF, size);
8435         }
8436
8437         /* Action data size. */
8438         params->action_data_size = l->action_data_size_max;
8439
8440         /* Maximum number of keys. */
8441         params->n_keys_max = l->size;
8442
8443         /* Timeout. */
8444         params->key_timeout = l->timeout;
8445
8446         return params;
8447
8448 error:
8449         learner_params_free(params);
8450         return NULL;
8451 }
8452
8453 static void
8454 learner_build_free(struct rte_swx_pipeline *p)
8455 {
8456         uint32_t i;
8457
8458         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8459                 struct thread *t = &p->threads[i];
8460                 uint32_t j;
8461
8462                 if (!t->learners)
8463                         continue;
8464
8465                 for (j = 0; j < p->n_learners; j++) {
8466                         struct learner_runtime *r = &t->learners[j];
8467
8468                         free(r->mailbox);
8469                 }
8470
8471                 free(t->learners);
8472                 t->learners = NULL;
8473         }
8474
8475         if (p->learner_stats) {
8476                 for (i = 0; i < p->n_learners; i++)
8477                         free(p->learner_stats[i].n_pkts_action);
8478
8479                 free(p->learner_stats);
8480         }
8481 }
8482
8483 static int
8484 learner_build(struct rte_swx_pipeline *p)
8485 {
8486         uint32_t i;
8487         int status = 0;
8488
8489         /* Per pipeline: learner statistics. */
8490         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8491         CHECK(p->learner_stats, ENOMEM);
8492
8493         for (i = 0; i < p->n_learners; i++) {
8494                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8495                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8496         }
8497
8498         /* Per thread: learner run-time. */
8499         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8500                 struct thread *t = &p->threads[i];
8501                 struct learner *l;
8502
8503                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8504                 if (!t->learners) {
8505                         status = -ENOMEM;
8506                         goto error;
8507                 }
8508
8509                 TAILQ_FOREACH(l, &p->learners, node) {
8510                         struct learner_runtime *r = &t->learners[l->id];
8511                         uint64_t size;
8512
8513                         /* r->mailbox. */
8514                         size = rte_swx_table_learner_mailbox_size_get();
8515                         if (size) {
8516                                 r->mailbox = calloc(1, size);
8517                                 if (!r->mailbox) {
8518                                         status = -ENOMEM;
8519                                         goto error;
8520                                 }
8521                         }
8522
8523                         /* r->key. */
8524                         r->key = l->header ?
8525                                 &t->structs[l->header->struct_id] :
8526                                 &t->structs[p->metadata_struct_id];
8527                 }
8528         }
8529
8530         return 0;
8531
8532 error:
8533         learner_build_free(p);
8534         return status;
8535 }
8536
8537 static void
8538 learner_free(struct rte_swx_pipeline *p)
8539 {
8540         learner_build_free(p);
8541
8542         /* Learner tables. */
8543         for ( ; ; ) {
8544                 struct learner *l;
8545
8546                 l = TAILQ_FIRST(&p->learners);
8547                 if (!l)
8548                         break;
8549
8550                 TAILQ_REMOVE(&p->learners, l, node);
8551                 free(l->fields);
8552                 free(l->actions);
8553                 free(l->default_action_data);
8554                 free(l);
8555         }
8556 }
8557
8558 /*
8559  * Table state.
8560  */
8561 static int
8562 table_state_build(struct rte_swx_pipeline *p)
8563 {
8564         struct table *table;
8565         struct selector *s;
8566         struct learner *l;
8567
8568         p->table_state = calloc(p->n_tables + p->n_selectors + p->n_learners,
8569                                 sizeof(struct rte_swx_table_state));
8570         CHECK(p->table_state, ENOMEM);
8571
8572         TAILQ_FOREACH(table, &p->tables, node) {
8573                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8574
8575                 if (table->type) {
8576                         struct rte_swx_table_params *params;
8577
8578                         /* ts->obj. */
8579                         params = table_params_get(table);
8580                         CHECK(params, ENOMEM);
8581
8582                         ts->obj = table->type->ops.create(params,
8583                                 NULL,
8584                                 table->args,
8585                                 p->numa_node);
8586
8587                         table_params_free(params);
8588                         CHECK(ts->obj, ENODEV);
8589                 }
8590
8591                 /* ts->default_action_data. */
8592                 if (table->action_data_size_max) {
8593                         ts->default_action_data =
8594                                 malloc(table->action_data_size_max);
8595                         CHECK(ts->default_action_data, ENOMEM);
8596
8597                         memcpy(ts->default_action_data,
8598                                table->default_action_data,
8599                                table->action_data_size_max);
8600                 }
8601
8602                 /* ts->default_action_id. */
8603                 ts->default_action_id = table->default_action->id;
8604         }
8605
8606         TAILQ_FOREACH(s, &p->selectors, node) {
8607                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8608                 struct rte_swx_table_selector_params *params;
8609
8610                 /* ts->obj. */
8611                 params = selector_table_params_get(s);
8612                 CHECK(params, ENOMEM);
8613
8614                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8615
8616                 selector_params_free(params);
8617                 CHECK(ts->obj, ENODEV);
8618         }
8619
8620         TAILQ_FOREACH(l, &p->learners, node) {
8621                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8622                         p->n_selectors + l->id];
8623                 struct rte_swx_table_learner_params *params;
8624
8625                 /* ts->obj. */
8626                 params = learner_params_get(l);
8627                 CHECK(params, ENOMEM);
8628
8629                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8630                 learner_params_free(params);
8631                 CHECK(ts->obj, ENODEV);
8632
8633                 /* ts->default_action_data. */
8634                 if (l->action_data_size_max) {
8635                         ts->default_action_data = malloc(l->action_data_size_max);
8636                         CHECK(ts->default_action_data, ENOMEM);
8637
8638                         memcpy(ts->default_action_data,
8639                                l->default_action_data,
8640                                l->action_data_size_max);
8641                 }
8642
8643                 /* ts->default_action_id. */
8644                 ts->default_action_id = l->default_action->id;
8645         }
8646
8647         return 0;
8648 }
8649
8650 static void
8651 table_state_build_free(struct rte_swx_pipeline *p)
8652 {
8653         uint32_t i;
8654
8655         if (!p->table_state)
8656                 return;
8657
8658         for (i = 0; i < p->n_tables; i++) {
8659                 struct rte_swx_table_state *ts = &p->table_state[i];
8660                 struct table *table = table_find_by_id(p, i);
8661
8662                 /* ts->obj. */
8663                 if (table->type && ts->obj)
8664                         table->type->ops.free(ts->obj);
8665
8666                 /* ts->default_action_data. */
8667                 free(ts->default_action_data);
8668         }
8669
8670         for (i = 0; i < p->n_selectors; i++) {
8671                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8672
8673                 /* ts->obj. */
8674                 if (ts->obj)
8675                         rte_swx_table_selector_free(ts->obj);
8676         }
8677
8678         for (i = 0; i < p->n_learners; i++) {
8679                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8680
8681                 /* ts->obj. */
8682                 if (ts->obj)
8683                         rte_swx_table_learner_free(ts->obj);
8684
8685                 /* ts->default_action_data. */
8686                 free(ts->default_action_data);
8687         }
8688
8689         free(p->table_state);
8690         p->table_state = NULL;
8691 }
8692
8693 static void
8694 table_state_free(struct rte_swx_pipeline *p)
8695 {
8696         table_state_build_free(p);
8697 }
8698
8699 /*
8700  * Register array.
8701  */
8702 static struct regarray *
8703 regarray_find(struct rte_swx_pipeline *p, const char *name)
8704 {
8705         struct regarray *elem;
8706
8707         TAILQ_FOREACH(elem, &p->regarrays, node)
8708                 if (!strcmp(elem->name, name))
8709                         return elem;
8710
8711         return NULL;
8712 }
8713
8714 static struct regarray *
8715 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8716 {
8717         struct regarray *elem = NULL;
8718
8719         TAILQ_FOREACH(elem, &p->regarrays, node)
8720                 if (elem->id == id)
8721                         return elem;
8722
8723         return NULL;
8724 }
8725
8726 int
8727 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8728                               const char *name,
8729                               uint32_t size,
8730                               uint64_t init_val)
8731 {
8732         struct regarray *r;
8733
8734         CHECK(p, EINVAL);
8735
8736         CHECK_NAME(name, EINVAL);
8737         CHECK(!regarray_find(p, name), EEXIST);
8738
8739         CHECK(size, EINVAL);
8740         size = rte_align32pow2(size);
8741
8742         /* Memory allocation. */
8743         r = calloc(1, sizeof(struct regarray));
8744         CHECK(r, ENOMEM);
8745
8746         /* Node initialization. */
8747         strcpy(r->name, name);
8748         r->init_val = init_val;
8749         r->size = size;
8750         r->id = p->n_regarrays;
8751
8752         /* Node add to tailq. */
8753         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8754         p->n_regarrays++;
8755
8756         return 0;
8757 }
8758
8759 static int
8760 regarray_build(struct rte_swx_pipeline *p)
8761 {
8762         struct regarray *regarray;
8763
8764         if (!p->n_regarrays)
8765                 return 0;
8766
8767         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8768         CHECK(p->regarray_runtime, ENOMEM);
8769
8770         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8771                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8772                 uint32_t i;
8773
8774                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8775                                          RTE_CACHE_LINE_SIZE,
8776                                          p->numa_node);
8777                 CHECK(r->regarray, ENOMEM);
8778
8779                 if (regarray->init_val)
8780                         for (i = 0; i < regarray->size; i++)
8781                                 r->regarray[i] = regarray->init_val;
8782
8783                 r->size_mask = regarray->size - 1;
8784         }
8785
8786         return 0;
8787 }
8788
8789 static void
8790 regarray_build_free(struct rte_swx_pipeline *p)
8791 {
8792         uint32_t i;
8793
8794         if (!p->regarray_runtime)
8795                 return;
8796
8797         for (i = 0; i < p->n_regarrays; i++) {
8798                 struct regarray *regarray = regarray_find_by_id(p, i);
8799                 struct regarray_runtime *r = &p->regarray_runtime[i];
8800
8801                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8802         }
8803
8804         free(p->regarray_runtime);
8805         p->regarray_runtime = NULL;
8806 }
8807
8808 static void
8809 regarray_free(struct rte_swx_pipeline *p)
8810 {
8811         regarray_build_free(p);
8812
8813         for ( ; ; ) {
8814                 struct regarray *elem;
8815
8816                 elem = TAILQ_FIRST(&p->regarrays);
8817                 if (!elem)
8818                         break;
8819
8820                 TAILQ_REMOVE(&p->regarrays, elem, node);
8821                 free(elem);
8822         }
8823 }
8824
8825 /*
8826  * Meter array.
8827  */
8828 static struct meter_profile *
8829 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8830 {
8831         struct meter_profile *elem;
8832
8833         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8834                 if (!strcmp(elem->name, name))
8835                         return elem;
8836
8837         return NULL;
8838 }
8839
8840 static struct metarray *
8841 metarray_find(struct rte_swx_pipeline *p, const char *name)
8842 {
8843         struct metarray *elem;
8844
8845         TAILQ_FOREACH(elem, &p->metarrays, node)
8846                 if (!strcmp(elem->name, name))
8847                         return elem;
8848
8849         return NULL;
8850 }
8851
8852 static struct metarray *
8853 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8854 {
8855         struct metarray *elem = NULL;
8856
8857         TAILQ_FOREACH(elem, &p->metarrays, node)
8858                 if (elem->id == id)
8859                         return elem;
8860
8861         return NULL;
8862 }
8863
8864 int
8865 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8866                                  const char *name,
8867                                  uint32_t size)
8868 {
8869         struct metarray *m;
8870
8871         CHECK(p, EINVAL);
8872
8873         CHECK_NAME(name, EINVAL);
8874         CHECK(!metarray_find(p, name), EEXIST);
8875
8876         CHECK(size, EINVAL);
8877         size = rte_align32pow2(size);
8878
8879         /* Memory allocation. */
8880         m = calloc(1, sizeof(struct metarray));
8881         CHECK(m, ENOMEM);
8882
8883         /* Node initialization. */
8884         strcpy(m->name, name);
8885         m->size = size;
8886         m->id = p->n_metarrays;
8887
8888         /* Node add to tailq. */
8889         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8890         p->n_metarrays++;
8891
8892         return 0;
8893 }
8894
8895 struct meter_profile meter_profile_default = {
8896         .node = {0},
8897         .name = "",
8898         .params = {0},
8899
8900         .profile = {
8901                 .cbs = 10000,
8902                 .pbs = 10000,
8903                 .cir_period = 1,
8904                 .cir_bytes_per_period = 1,
8905                 .pir_period = 1,
8906                 .pir_bytes_per_period = 1,
8907         },
8908
8909         .n_users = 0,
8910 };
8911
8912 static void
8913 meter_init(struct meter *m)
8914 {
8915         memset(m, 0, sizeof(struct meter));
8916         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8917         m->profile = &meter_profile_default;
8918         m->color_mask = RTE_COLOR_GREEN;
8919
8920         meter_profile_default.n_users++;
8921 }
8922
8923 static int
8924 metarray_build(struct rte_swx_pipeline *p)
8925 {
8926         struct metarray *m;
8927
8928         if (!p->n_metarrays)
8929                 return 0;
8930
8931         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8932         CHECK(p->metarray_runtime, ENOMEM);
8933
8934         TAILQ_FOREACH(m, &p->metarrays, node) {
8935                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8936                 uint32_t i;
8937
8938                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8939                                          RTE_CACHE_LINE_SIZE,
8940                                          p->numa_node);
8941                 CHECK(r->metarray, ENOMEM);
8942
8943                 for (i = 0; i < m->size; i++)
8944                         meter_init(&r->metarray[i]);
8945
8946                 r->size_mask = m->size - 1;
8947         }
8948
8949         return 0;
8950 }
8951
8952 static void
8953 metarray_build_free(struct rte_swx_pipeline *p)
8954 {
8955         uint32_t i;
8956
8957         if (!p->metarray_runtime)
8958                 return;
8959
8960         for (i = 0; i < p->n_metarrays; i++) {
8961                 struct metarray *m = metarray_find_by_id(p, i);
8962                 struct metarray_runtime *r = &p->metarray_runtime[i];
8963
8964                 env_free(r->metarray, m->size * sizeof(struct meter));
8965         }
8966
8967         free(p->metarray_runtime);
8968         p->metarray_runtime = NULL;
8969 }
8970
8971 static void
8972 metarray_free(struct rte_swx_pipeline *p)
8973 {
8974         metarray_build_free(p);
8975
8976         /* Meter arrays. */
8977         for ( ; ; ) {
8978                 struct metarray *elem;
8979
8980                 elem = TAILQ_FIRST(&p->metarrays);
8981                 if (!elem)
8982                         break;
8983
8984                 TAILQ_REMOVE(&p->metarrays, elem, node);
8985                 free(elem);
8986         }
8987
8988         /* Meter profiles. */
8989         for ( ; ; ) {
8990                 struct meter_profile *elem;
8991
8992                 elem = TAILQ_FIRST(&p->meter_profiles);
8993                 if (!elem)
8994                         break;
8995
8996                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8997                 free(elem);
8998         }
8999 }
9000
9001 /*
9002  * Pipeline.
9003  */
9004 void
9005 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9006 {
9007         void *lib;
9008
9009         if (!p)
9010                 return;
9011
9012         lib = p->lib;
9013
9014         free(p->instruction_data);
9015         free(p->instructions);
9016
9017         metarray_free(p);
9018         regarray_free(p);
9019         table_state_free(p);
9020         learner_free(p);
9021         selector_free(p);
9022         table_free(p);
9023         action_free(p);
9024         instruction_table_free(p);
9025         metadata_free(p);
9026         header_free(p);
9027         extern_func_free(p);
9028         extern_obj_free(p);
9029         port_out_free(p);
9030         port_in_free(p);
9031         struct_free(p);
9032
9033         free(p);
9034
9035         if (lib)
9036                 dlclose(lib);
9037 }
9038
9039 static int
9040 port_in_types_register(struct rte_swx_pipeline *p)
9041 {
9042         int status;
9043
9044         status = rte_swx_pipeline_port_in_type_register(p,
9045                 "ethdev",
9046                 &rte_swx_port_ethdev_reader_ops);
9047         if (status)
9048                 return status;
9049
9050         status = rte_swx_pipeline_port_in_type_register(p,
9051                 "ring",
9052                 &rte_swx_port_ring_reader_ops);
9053         if (status)
9054                 return status;
9055
9056 #ifdef RTE_PORT_PCAP
9057         status = rte_swx_pipeline_port_in_type_register(p,
9058                 "source",
9059                 &rte_swx_port_source_ops);
9060         if (status)
9061                 return status;
9062 #endif
9063
9064         status = rte_swx_pipeline_port_in_type_register(p,
9065                 "fd",
9066                 &rte_swx_port_fd_reader_ops);
9067         if (status)
9068                 return status;
9069
9070         return 0;
9071 }
9072
9073 static int
9074 port_out_types_register(struct rte_swx_pipeline *p)
9075 {
9076         int status;
9077
9078         status = rte_swx_pipeline_port_out_type_register(p,
9079                 "ethdev",
9080                 &rte_swx_port_ethdev_writer_ops);
9081         if (status)
9082                 return status;
9083
9084         status = rte_swx_pipeline_port_out_type_register(p,
9085                 "ring",
9086                 &rte_swx_port_ring_writer_ops);
9087         if (status)
9088                 return status;
9089
9090         status = rte_swx_pipeline_port_out_type_register(p,
9091                 "sink",
9092                 &rte_swx_port_sink_ops);
9093         if (status)
9094                 return status;
9095
9096         status = rte_swx_pipeline_port_out_type_register(p,
9097                 "fd",
9098                 &rte_swx_port_fd_writer_ops);
9099         if (status)
9100                 return status;
9101
9102         return 0;
9103 }
9104
9105 static int
9106 table_types_register(struct rte_swx_pipeline *p)
9107 {
9108         int status;
9109
9110         status = rte_swx_pipeline_table_type_register(p,
9111                 "exact",
9112                 RTE_SWX_TABLE_MATCH_EXACT,
9113                 &rte_swx_table_exact_match_ops);
9114         if (status)
9115                 return status;
9116
9117         status = rte_swx_pipeline_table_type_register(p,
9118                 "wildcard",
9119                 RTE_SWX_TABLE_MATCH_WILDCARD,
9120                 &rte_swx_table_wildcard_match_ops);
9121         if (status)
9122                 return status;
9123
9124         return 0;
9125 }
9126
9127 int
9128 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9129 {
9130         struct rte_swx_pipeline *pipeline = NULL;
9131         int status = 0;
9132
9133         /* Check input parameters. */
9134         CHECK(p, EINVAL);
9135
9136         /* Memory allocation. */
9137         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9138         if (!pipeline) {
9139                 status = -ENOMEM;
9140                 goto error;
9141         }
9142
9143         /* Initialization. */
9144         TAILQ_INIT(&pipeline->struct_types);
9145         TAILQ_INIT(&pipeline->port_in_types);
9146         TAILQ_INIT(&pipeline->ports_in);
9147         TAILQ_INIT(&pipeline->port_out_types);
9148         TAILQ_INIT(&pipeline->ports_out);
9149         TAILQ_INIT(&pipeline->extern_types);
9150         TAILQ_INIT(&pipeline->extern_objs);
9151         TAILQ_INIT(&pipeline->extern_funcs);
9152         TAILQ_INIT(&pipeline->headers);
9153         TAILQ_INIT(&pipeline->actions);
9154         TAILQ_INIT(&pipeline->table_types);
9155         TAILQ_INIT(&pipeline->tables);
9156         TAILQ_INIT(&pipeline->selectors);
9157         TAILQ_INIT(&pipeline->learners);
9158         TAILQ_INIT(&pipeline->regarrays);
9159         TAILQ_INIT(&pipeline->meter_profiles);
9160         TAILQ_INIT(&pipeline->metarrays);
9161
9162         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9163         pipeline->numa_node = numa_node;
9164
9165         status = port_in_types_register(pipeline);
9166         if (status)
9167                 goto error;
9168
9169         status = port_out_types_register(pipeline);
9170         if (status)
9171                 goto error;
9172
9173         status = table_types_register(pipeline);
9174         if (status)
9175                 goto error;
9176
9177         *p = pipeline;
9178         return 0;
9179
9180 error:
9181         rte_swx_pipeline_free(pipeline);
9182         return status;
9183 }
9184
9185 int
9186 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9187                                      const char **instructions,
9188                                      uint32_t n_instructions)
9189 {
9190         int err;
9191         uint32_t i;
9192
9193         err = instruction_config(p, NULL, instructions, n_instructions);
9194         if (err)
9195                 return err;
9196
9197         /* Thread instruction pointer reset. */
9198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9199                 struct thread *t = &p->threads[i];
9200
9201                 thread_ip_reset(p, t);
9202         }
9203
9204         return 0;
9205 }
9206
9207 static int
9208 pipeline_compile(struct rte_swx_pipeline *p);
9209
9210 int
9211 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9212 {
9213         struct rte_swx_port_sink_params drop_port_params = {
9214                 .file_name = NULL,
9215         };
9216         int status;
9217
9218         CHECK(p, EINVAL);
9219         CHECK(p->build_done == 0, EEXIST);
9220
9221         status = port_in_build(p);
9222         if (status)
9223                 goto error;
9224
9225         /* Drop port. */
9226         status = rte_swx_pipeline_port_out_config(p,
9227                                                   p->n_ports_out,
9228                                                   "sink",
9229                                                   &drop_port_params);
9230         if (status)
9231                 goto error;
9232
9233         status = port_out_build(p);
9234         if (status)
9235                 goto error;
9236
9237         status = struct_build(p);
9238         if (status)
9239                 goto error;
9240
9241         status = extern_obj_build(p);
9242         if (status)
9243                 goto error;
9244
9245         status = extern_func_build(p);
9246         if (status)
9247                 goto error;
9248
9249         status = header_build(p);
9250         if (status)
9251                 goto error;
9252
9253         status = metadata_build(p);
9254         if (status)
9255                 goto error;
9256
9257         status = instruction_table_build(p);
9258         if (status)
9259                 goto error;
9260
9261         status = action_build(p);
9262         if (status)
9263                 goto error;
9264
9265         status = table_build(p);
9266         if (status)
9267                 goto error;
9268
9269         status = selector_build(p);
9270         if (status)
9271                 goto error;
9272
9273         status = learner_build(p);
9274         if (status)
9275                 goto error;
9276
9277         status = table_state_build(p);
9278         if (status)
9279                 goto error;
9280
9281         status = regarray_build(p);
9282         if (status)
9283                 goto error;
9284
9285         status = metarray_build(p);
9286         if (status)
9287                 goto error;
9288
9289         p->build_done = 1;
9290
9291         pipeline_compile(p);
9292
9293         return 0;
9294
9295 error:
9296         metarray_build_free(p);
9297         regarray_build_free(p);
9298         table_state_build_free(p);
9299         learner_build_free(p);
9300         selector_build_free(p);
9301         table_build_free(p);
9302         action_build_free(p);
9303         instruction_table_build_free(p);
9304         metadata_build_free(p);
9305         header_build_free(p);
9306         extern_func_build_free(p);
9307         extern_obj_build_free(p);
9308         port_out_build_free(p);
9309         port_in_build_free(p);
9310         struct_build_free(p);
9311
9312         return status;
9313 }
9314
9315 void
9316 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9317 {
9318         uint32_t i;
9319
9320         for (i = 0; i < n_instructions; i++)
9321                 instr_exec(p);
9322 }
9323
9324 void
9325 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9326 {
9327         uint32_t i;
9328
9329         for (i = 0; i < p->n_ports_out; i++) {
9330                 struct port_out_runtime *port = &p->out[i];
9331
9332                 if (port->flush)
9333                         port->flush(port->obj);
9334         }
9335 }
9336
9337 /*
9338  * Control.
9339  */
9340 int
9341 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9342                               struct rte_swx_ctl_pipeline_info *pipeline)
9343 {
9344         struct action *action;
9345         struct table *table;
9346         uint32_t n_actions = 0, n_tables = 0;
9347
9348         if (!p || !pipeline)
9349                 return -EINVAL;
9350
9351         TAILQ_FOREACH(action, &p->actions, node)
9352                 n_actions++;
9353
9354         TAILQ_FOREACH(table, &p->tables, node)
9355                 n_tables++;
9356
9357         pipeline->n_ports_in = p->n_ports_in;
9358         pipeline->n_ports_out = p->n_ports_out;
9359         pipeline->n_actions = n_actions;
9360         pipeline->n_tables = n_tables;
9361         pipeline->n_selectors = p->n_selectors;
9362         pipeline->n_learners = p->n_learners;
9363         pipeline->n_regarrays = p->n_regarrays;
9364         pipeline->n_metarrays = p->n_metarrays;
9365
9366         return 0;
9367 }
9368
9369 int
9370 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9371 {
9372         if (!p || !numa_node)
9373                 return -EINVAL;
9374
9375         *numa_node = p->numa_node;
9376         return 0;
9377 }
9378
9379 int
9380 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9381                             uint32_t action_id,
9382                             struct rte_swx_ctl_action_info *action)
9383 {
9384         struct action *a = NULL;
9385
9386         if (!p || (action_id >= p->n_actions) || !action)
9387                 return -EINVAL;
9388
9389         a = action_find_by_id(p, action_id);
9390         if (!a)
9391                 return -EINVAL;
9392
9393         strcpy(action->name, a->name);
9394         action->n_args = a->st ? a->st->n_fields : 0;
9395         return 0;
9396 }
9397
9398 int
9399 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9400                                 uint32_t action_id,
9401                                 uint32_t action_arg_id,
9402                                 struct rte_swx_ctl_action_arg_info *action_arg)
9403 {
9404         struct action *a = NULL;
9405         struct field *arg = NULL;
9406
9407         if (!p || (action_id >= p->n_actions) || !action_arg)
9408                 return -EINVAL;
9409
9410         a = action_find_by_id(p, action_id);
9411         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9412                 return -EINVAL;
9413
9414         arg = &a->st->fields[action_arg_id];
9415         strcpy(action_arg->name, arg->name);
9416         action_arg->n_bits = arg->n_bits;
9417         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9418
9419         return 0;
9420 }
9421
9422 int
9423 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9424                            uint32_t table_id,
9425                            struct rte_swx_ctl_table_info *table)
9426 {
9427         struct table *t = NULL;
9428
9429         if (!p || !table)
9430                 return -EINVAL;
9431
9432         t = table_find_by_id(p, table_id);
9433         if (!t)
9434                 return -EINVAL;
9435
9436         strcpy(table->name, t->name);
9437         strcpy(table->args, t->args);
9438         table->n_match_fields = t->n_fields;
9439         table->n_actions = t->n_actions;
9440         table->default_action_is_const = t->default_action_is_const;
9441         table->size = t->size;
9442         return 0;
9443 }
9444
9445 int
9446 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9447         uint32_t table_id,
9448         uint32_t match_field_id,
9449         struct rte_swx_ctl_table_match_field_info *match_field)
9450 {
9451         struct table *t;
9452         struct match_field *f;
9453
9454         if (!p || (table_id >= p->n_tables) || !match_field)
9455                 return -EINVAL;
9456
9457         t = table_find_by_id(p, table_id);
9458         if (!t || (match_field_id >= t->n_fields))
9459                 return -EINVAL;
9460
9461         f = &t->fields[match_field_id];
9462         match_field->match_type = f->match_type;
9463         match_field->is_header = t->header ? 1 : 0;
9464         match_field->n_bits = f->field->n_bits;
9465         match_field->offset = f->field->offset;
9466
9467         return 0;
9468 }
9469
9470 int
9471 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9472         uint32_t table_id,
9473         uint32_t table_action_id,
9474         struct rte_swx_ctl_table_action_info *table_action)
9475 {
9476         struct table *t;
9477
9478         if (!p || (table_id >= p->n_tables) || !table_action)
9479                 return -EINVAL;
9480
9481         t = table_find_by_id(p, table_id);
9482         if (!t || (table_action_id >= t->n_actions))
9483                 return -EINVAL;
9484
9485         table_action->action_id = t->actions[table_action_id]->id;
9486
9487         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9488         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9489
9490         return 0;
9491 }
9492
9493 int
9494 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9495                           uint32_t table_id,
9496                           struct rte_swx_table_ops *table_ops,
9497                           int *is_stub)
9498 {
9499         struct table *t;
9500
9501         if (!p || (table_id >= p->n_tables))
9502                 return -EINVAL;
9503
9504         t = table_find_by_id(p, table_id);
9505         if (!t)
9506                 return -EINVAL;
9507
9508         if (t->type) {
9509                 if (table_ops)
9510                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9511                 *is_stub = 0;
9512         } else {
9513                 *is_stub = 1;
9514         }
9515
9516         return 0;
9517 }
9518
9519 int
9520 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9521                               uint32_t selector_id,
9522                               struct rte_swx_ctl_selector_info *selector)
9523 {
9524         struct selector *s = NULL;
9525
9526         if (!p || !selector)
9527                 return -EINVAL;
9528
9529         s = selector_find_by_id(p, selector_id);
9530         if (!s)
9531                 return -EINVAL;
9532
9533         strcpy(selector->name, s->name);
9534
9535         selector->n_selector_fields = s->n_selector_fields;
9536         selector->n_groups_max = s->n_groups_max;
9537         selector->n_members_per_group_max = s->n_members_per_group_max;
9538
9539         return 0;
9540 }
9541
9542 int
9543 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9544          uint32_t selector_id,
9545          struct rte_swx_ctl_table_match_field_info *field)
9546 {
9547         struct selector *s;
9548
9549         if (!p || (selector_id >= p->n_selectors) || !field)
9550                 return -EINVAL;
9551
9552         s = selector_find_by_id(p, selector_id);
9553         if (!s)
9554                 return -EINVAL;
9555
9556         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9557         field->is_header = 0;
9558         field->n_bits = s->group_id_field->n_bits;
9559         field->offset = s->group_id_field->offset;
9560
9561         return 0;
9562 }
9563
9564 int
9565 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9566          uint32_t selector_id,
9567          uint32_t selector_field_id,
9568          struct rte_swx_ctl_table_match_field_info *field)
9569 {
9570         struct selector *s;
9571         struct field *f;
9572
9573         if (!p || (selector_id >= p->n_selectors) || !field)
9574                 return -EINVAL;
9575
9576         s = selector_find_by_id(p, selector_id);
9577         if (!s || (selector_field_id >= s->n_selector_fields))
9578                 return -EINVAL;
9579
9580         f = s->selector_fields[selector_field_id];
9581         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9582         field->is_header = s->selector_header ? 1 : 0;
9583         field->n_bits = f->n_bits;
9584         field->offset = f->offset;
9585
9586         return 0;
9587 }
9588
9589 int
9590 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9591          uint32_t selector_id,
9592          struct rte_swx_ctl_table_match_field_info *field)
9593 {
9594         struct selector *s;
9595
9596         if (!p || (selector_id >= p->n_selectors) || !field)
9597                 return -EINVAL;
9598
9599         s = selector_find_by_id(p, selector_id);
9600         if (!s)
9601                 return -EINVAL;
9602
9603         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9604         field->is_header = 0;
9605         field->n_bits = s->member_id_field->n_bits;
9606         field->offset = s->member_id_field->offset;
9607
9608         return 0;
9609 }
9610
9611 int
9612 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9613                              uint32_t learner_id,
9614                              struct rte_swx_ctl_learner_info *learner)
9615 {
9616         struct learner *l = NULL;
9617
9618         if (!p || !learner)
9619                 return -EINVAL;
9620
9621         l = learner_find_by_id(p, learner_id);
9622         if (!l)
9623                 return -EINVAL;
9624
9625         strcpy(learner->name, l->name);
9626
9627         learner->n_match_fields = l->n_fields;
9628         learner->n_actions = l->n_actions;
9629         learner->default_action_is_const = l->default_action_is_const;
9630         learner->size = l->size;
9631
9632         return 0;
9633 }
9634
9635 int
9636 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9637                                          uint32_t learner_id,
9638                                          uint32_t match_field_id,
9639                                          struct rte_swx_ctl_table_match_field_info *match_field)
9640 {
9641         struct learner *l;
9642         struct field *f;
9643
9644         if (!p || (learner_id >= p->n_learners) || !match_field)
9645                 return -EINVAL;
9646
9647         l = learner_find_by_id(p, learner_id);
9648         if (!l || (match_field_id >= l->n_fields))
9649                 return -EINVAL;
9650
9651         f = l->fields[match_field_id];
9652         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9653         match_field->is_header = l->header ? 1 : 0;
9654         match_field->n_bits = f->n_bits;
9655         match_field->offset = f->offset;
9656
9657         return 0;
9658 }
9659
9660 int
9661 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9662                                     uint32_t learner_id,
9663                                     uint32_t learner_action_id,
9664                                     struct rte_swx_ctl_table_action_info *learner_action)
9665 {
9666         struct learner *l;
9667
9668         if (!p || (learner_id >= p->n_learners) || !learner_action)
9669                 return -EINVAL;
9670
9671         l = learner_find_by_id(p, learner_id);
9672         if (!l || (learner_action_id >= l->n_actions))
9673                 return -EINVAL;
9674
9675         learner_action->action_id = l->actions[learner_action_id]->id;
9676
9677         learner_action->action_is_for_table_entries =
9678                 l->action_is_for_table_entries[learner_action_id];
9679
9680         learner_action->action_is_for_default_entry =
9681                 l->action_is_for_default_entry[learner_action_id];
9682
9683         return 0;
9684 }
9685
9686 int
9687 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9688                                  struct rte_swx_table_state **table_state)
9689 {
9690         if (!p || !table_state || !p->build_done)
9691                 return -EINVAL;
9692
9693         *table_state = p->table_state;
9694         return 0;
9695 }
9696
9697 int
9698 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9699                                  struct rte_swx_table_state *table_state)
9700 {
9701         if (!p || !table_state || !p->build_done)
9702                 return -EINVAL;
9703
9704         p->table_state = table_state;
9705         return 0;
9706 }
9707
9708 int
9709 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9710                                         uint32_t port_id,
9711                                         struct rte_swx_port_in_stats *stats)
9712 {
9713         struct port_in *port;
9714
9715         if (!p || !stats)
9716                 return -EINVAL;
9717
9718         port = port_in_find(p, port_id);
9719         if (!port)
9720                 return -EINVAL;
9721
9722         port->type->ops.stats_read(port->obj, stats);
9723         return 0;
9724 }
9725
9726 int
9727 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9728                                          uint32_t port_id,
9729                                          struct rte_swx_port_out_stats *stats)
9730 {
9731         struct port_out *port;
9732
9733         if (!p || !stats)
9734                 return -EINVAL;
9735
9736         port = port_out_find(p, port_id);
9737         if (!port)
9738                 return -EINVAL;
9739
9740         port->type->ops.stats_read(port->obj, stats);
9741         return 0;
9742 }
9743
9744 int
9745 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9746                                       const char *table_name,
9747                                       struct rte_swx_table_stats *stats)
9748 {
9749         struct table *table;
9750         struct table_statistics *table_stats;
9751
9752         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9753                 return -EINVAL;
9754
9755         table = table_find(p, table_name);
9756         if (!table)
9757                 return -EINVAL;
9758
9759         table_stats = &p->table_stats[table->id];
9760
9761         memcpy(stats->n_pkts_action,
9762                table_stats->n_pkts_action,
9763                p->n_actions * sizeof(uint64_t));
9764
9765         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9766         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9767
9768         return 0;
9769 }
9770
9771 int
9772 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9773         const char *selector_name,
9774         struct rte_swx_pipeline_selector_stats *stats)
9775 {
9776         struct selector *s;
9777
9778         if (!p || !selector_name || !selector_name[0] || !stats)
9779                 return -EINVAL;
9780
9781         s = selector_find(p, selector_name);
9782         if (!s)
9783                 return -EINVAL;
9784
9785         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9786
9787         return 0;
9788 }
9789
9790 int
9791 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9792                                         const char *learner_name,
9793                                         struct rte_swx_learner_stats *stats)
9794 {
9795         struct learner *l;
9796         struct learner_statistics *learner_stats;
9797
9798         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9799                 return -EINVAL;
9800
9801         l = learner_find(p, learner_name);
9802         if (!l)
9803                 return -EINVAL;
9804
9805         learner_stats = &p->learner_stats[l->id];
9806
9807         memcpy(stats->n_pkts_action,
9808                learner_stats->n_pkts_action,
9809                p->n_actions * sizeof(uint64_t));
9810
9811         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9812         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9813
9814         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9815         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9816
9817         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9818
9819         return 0;
9820 }
9821
9822 int
9823 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9824                               uint32_t regarray_id,
9825                               struct rte_swx_ctl_regarray_info *regarray)
9826 {
9827         struct regarray *r;
9828
9829         if (!p || !regarray)
9830                 return -EINVAL;
9831
9832         r = regarray_find_by_id(p, regarray_id);
9833         if (!r)
9834                 return -EINVAL;
9835
9836         strcpy(regarray->name, r->name);
9837         regarray->size = r->size;
9838         return 0;
9839 }
9840
9841 int
9842 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9843                                    const char *regarray_name,
9844                                    uint32_t regarray_index,
9845                                    uint64_t *value)
9846 {
9847         struct regarray *regarray;
9848         struct regarray_runtime *r;
9849
9850         if (!p || !regarray_name || !value)
9851                 return -EINVAL;
9852
9853         regarray = regarray_find(p, regarray_name);
9854         if (!regarray || (regarray_index >= regarray->size))
9855                 return -EINVAL;
9856
9857         r = &p->regarray_runtime[regarray->id];
9858         *value = r->regarray[regarray_index];
9859         return 0;
9860 }
9861
9862 int
9863 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9864                                    const char *regarray_name,
9865                                    uint32_t regarray_index,
9866                                    uint64_t value)
9867 {
9868         struct regarray *regarray;
9869         struct regarray_runtime *r;
9870
9871         if (!p || !regarray_name)
9872                 return -EINVAL;
9873
9874         regarray = regarray_find(p, regarray_name);
9875         if (!regarray || (regarray_index >= regarray->size))
9876                 return -EINVAL;
9877
9878         r = &p->regarray_runtime[regarray->id];
9879         r->regarray[regarray_index] = value;
9880         return 0;
9881 }
9882
9883 int
9884 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9885                               uint32_t metarray_id,
9886                               struct rte_swx_ctl_metarray_info *metarray)
9887 {
9888         struct metarray *m;
9889
9890         if (!p || !metarray)
9891                 return -EINVAL;
9892
9893         m = metarray_find_by_id(p, metarray_id);
9894         if (!m)
9895                 return -EINVAL;
9896
9897         strcpy(metarray->name, m->name);
9898         metarray->size = m->size;
9899         return 0;
9900 }
9901
9902 int
9903 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9904                               const char *name,
9905                               struct rte_meter_trtcm_params *params)
9906 {
9907         struct meter_profile *mp;
9908         int status;
9909
9910         CHECK(p, EINVAL);
9911         CHECK_NAME(name, EINVAL);
9912         CHECK(params, EINVAL);
9913         CHECK(!meter_profile_find(p, name), EEXIST);
9914
9915         /* Node allocation. */
9916         mp = calloc(1, sizeof(struct meter_profile));
9917         CHECK(mp, ENOMEM);
9918
9919         /* Node initialization. */
9920         strcpy(mp->name, name);
9921         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9922         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9923         if (status) {
9924                 free(mp);
9925                 CHECK(0, EINVAL);
9926         }
9927
9928         /* Node add to tailq. */
9929         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9930
9931         return 0;
9932 }
9933
9934 int
9935 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9936                                  const char *name)
9937 {
9938         struct meter_profile *mp;
9939
9940         CHECK(p, EINVAL);
9941         CHECK_NAME(name, EINVAL);
9942
9943         mp = meter_profile_find(p, name);
9944         CHECK(mp, EINVAL);
9945         CHECK(!mp->n_users, EBUSY);
9946
9947         /* Remove node from tailq. */
9948         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9949         free(mp);
9950
9951         return 0;
9952 }
9953
9954 int
9955 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9956                         const char *metarray_name,
9957                         uint32_t metarray_index)
9958 {
9959         struct meter_profile *mp_old;
9960         struct metarray *metarray;
9961         struct metarray_runtime *metarray_runtime;
9962         struct meter *m;
9963
9964         CHECK(p, EINVAL);
9965         CHECK_NAME(metarray_name, EINVAL);
9966
9967         metarray = metarray_find(p, metarray_name);
9968         CHECK(metarray, EINVAL);
9969         CHECK(metarray_index < metarray->size, EINVAL);
9970
9971         metarray_runtime = &p->metarray_runtime[metarray->id];
9972         m = &metarray_runtime->metarray[metarray_index];
9973         mp_old = m->profile;
9974
9975         meter_init(m);
9976
9977         mp_old->n_users--;
9978
9979         return 0;
9980 }
9981
9982 int
9983 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9984                       const char *metarray_name,
9985                       uint32_t metarray_index,
9986                       const char *profile_name)
9987 {
9988         struct meter_profile *mp, *mp_old;
9989         struct metarray *metarray;
9990         struct metarray_runtime *metarray_runtime;
9991         struct meter *m;
9992
9993         CHECK(p, EINVAL);
9994         CHECK_NAME(metarray_name, EINVAL);
9995
9996         metarray = metarray_find(p, metarray_name);
9997         CHECK(metarray, EINVAL);
9998         CHECK(metarray_index < metarray->size, EINVAL);
9999
10000         mp = meter_profile_find(p, profile_name);
10001         CHECK(mp, EINVAL);
10002
10003         metarray_runtime = &p->metarray_runtime[metarray->id];
10004         m = &metarray_runtime->metarray[metarray_index];
10005         mp_old = m->profile;
10006
10007         memset(m, 0, sizeof(struct meter));
10008         rte_meter_trtcm_config(&m->m, &mp->profile);
10009         m->profile = mp;
10010         m->color_mask = RTE_COLORS;
10011
10012         mp->n_users++;
10013         mp_old->n_users--;
10014
10015         return 0;
10016 }
10017
10018 int
10019 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10020                              const char *metarray_name,
10021                              uint32_t metarray_index,
10022                              struct rte_swx_ctl_meter_stats *stats)
10023 {
10024         struct metarray *metarray;
10025         struct metarray_runtime *metarray_runtime;
10026         struct meter *m;
10027
10028         CHECK(p, EINVAL);
10029         CHECK_NAME(metarray_name, EINVAL);
10030
10031         metarray = metarray_find(p, metarray_name);
10032         CHECK(metarray, EINVAL);
10033         CHECK(metarray_index < metarray->size, EINVAL);
10034
10035         CHECK(stats, EINVAL);
10036
10037         metarray_runtime = &p->metarray_runtime[metarray->id];
10038         m = &metarray_runtime->metarray[metarray_index];
10039
10040         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10041         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10042
10043         return 0;
10044 }
10045
10046 /*
10047  * Pipeline compilation.
10048  */
10049 static const char *
10050 instr_type_to_name(struct instruction *instr)
10051 {
10052         switch (instr->type) {
10053         case INSTR_RX: return "INSTR_RX";
10054
10055         case INSTR_TX: return "INSTR_TX";
10056         case INSTR_TX_I: return "INSTR_TX_I";
10057         case INSTR_DROP: return "INSTR_DROP";
10058
10059         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10060         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10061         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10062         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10063         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10064         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10065         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10066         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10067
10068         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10069
10070         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10071
10072         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10073         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10074         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10075         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10076         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10077         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10078         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10079         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10080         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10081
10082         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10083         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10084
10085         case INSTR_MOV: return "INSTR_MOV";
10086         case INSTR_MOV_MH: return "INSTR_MOV_MH";
10087         case INSTR_MOV_HM: return "INSTR_MOV_HM";
10088         case INSTR_MOV_HH: return "INSTR_MOV_HH";
10089         case INSTR_MOV_I: return "INSTR_MOV_I";
10090
10091         case INSTR_DMA_HT: return "INSTR_DMA_HT";
10092         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10093         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10094         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10095         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10096         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10097         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10098         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10099
10100         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10101         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10102         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10103         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10104         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10105         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10106
10107         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10108         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10109         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10110         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10111         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10112         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10113
10114         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10115         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10116         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10117         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10118
10119         case INSTR_ALU_AND: return "INSTR_ALU_AND";
10120         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10121         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10122         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10123         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10124
10125         case INSTR_ALU_OR: return "INSTR_ALU_OR";
10126         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10127         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10128         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10129         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10130
10131         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10132         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10133         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10134         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10135         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10136
10137         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10138         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10139         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10140         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10141         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10142         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10143
10144         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10145         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10146         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10147         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10148         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10149         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10150
10151         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10152         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10153         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10154
10155         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10156         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10157         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10158         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10159         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10160         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10161
10162         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10163         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10164         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10165         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10166         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10167         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10168         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10169         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10170         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10171
10172         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10173         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10174         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10175         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10176         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10177         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10178         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10179         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10180         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10181
10182         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10183         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10184         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10185
10186         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10187         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10188         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10189         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10190         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10191         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10192         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10193         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10194         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10195         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10196         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10197         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10198
10199         case INSTR_TABLE: return "INSTR_TABLE";
10200         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10201         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10202         case INSTR_LEARNER: return "INSTR_LEARNER";
10203         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10204
10205         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10206         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10207
10208         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10209         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10210
10211         case INSTR_JMP: return "INSTR_JMP";
10212         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10213         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10214         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10215         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10216         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10217         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10218         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10219         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10220         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10221         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10222         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10223         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10224         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10225         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10226         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10227         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10228         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10229         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10230         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10231         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10232         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10233         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10234         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10235         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10236         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10237         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10238         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10239         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10240
10241         case INSTR_RETURN: return "INSTR_RETURN";
10242
10243         default: return "INSTR_UNKNOWN";
10244         }
10245 }
10246
10247 typedef void
10248 (*instruction_export_t)(struct instruction *, FILE *);
10249
10250 static void
10251 instr_io_export(struct instruction *instr, FILE *f)
10252 {
10253         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10254
10255         /* n_io, n_io_imm, n_hdrs. */
10256         if (instr->type == INSTR_RX ||
10257             instr->type == INSTR_TX ||
10258             instr->type == INSTR_HDR_EXTRACT_M ||
10259             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10260                 n_io = 1;
10261
10262         if (instr->type == INSTR_TX_I)
10263                 n_io_imm = 1;
10264
10265         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10266                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10267
10268         if (instr->type == INSTR_HDR_EXTRACT_M ||
10269             instr->type == INSTR_HDR_LOOKAHEAD ||
10270             instr->type == INSTR_HDR_EMIT)
10271                 n_hdrs = 1;
10272
10273         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10274                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10275
10276         /* instr. */
10277         fprintf(f,
10278                 "\t{\n"
10279                 "\t\t.type = %s,\n",
10280                 instr_type_to_name(instr));
10281
10282         /* instr.io. */
10283         if (n_io || n_io_imm || n_hdrs)
10284                 fprintf(f,
10285                         "\t\t.io = {\n");
10286
10287         /* instr.io.io. */
10288         if (n_io)
10289                 fprintf(f,
10290                         "\t\t\t.io = {\n"
10291                         "\t\t\t\t.offset = %u,\n"
10292                         "\t\t\t\t.n_bits = %u,\n"
10293                         "\t\t\t},\n",
10294                         instr->io.io.offset,
10295                         instr->io.io.n_bits);
10296
10297         if (n_io_imm)
10298                 fprintf(f,
10299                         "\t\t\t.io = {\n"
10300                         "\t\t\t\t.val = %u,\n"
10301                         "\t\t\t},\n",
10302                         instr->io.io.val);
10303
10304         /* instr.io.hdr. */
10305         if (n_hdrs) {
10306                 fprintf(f,
10307                         "\t\t.hdr = {\n");
10308
10309                 /* instr.io.hdr.header_id. */
10310                 fprintf(f,
10311                         "\t\t\t.header_id = {");
10312
10313                 for (i = 0; i < n_hdrs; i++)
10314                         fprintf(f,
10315                                 "%u, ",
10316                                 instr->io.hdr.header_id[i]);
10317
10318                 fprintf(f,
10319                         "},\n");
10320
10321                 /* instr.io.hdr.struct_id. */
10322                 fprintf(f,
10323                         "\t\t\t.struct_id = {");
10324
10325                 for (i = 0; i < n_hdrs; i++)
10326                         fprintf(f,
10327                                 "%u, ",
10328                                 instr->io.hdr.struct_id[i]);
10329
10330                 fprintf(f,
10331                         "},\n");
10332
10333                 /* instr.io.hdr.n_bytes. */
10334                 fprintf(f,
10335                         "\t\t\t.n_bytes = {");
10336
10337                 for (i = 0; i < n_hdrs; i++)
10338                         fprintf(f,
10339                                 "%u, ",
10340                                 instr->io.hdr.n_bytes[i]);
10341
10342                 fprintf(f,
10343                         "},\n");
10344
10345                 /* instr.io.hdr - closing curly brace. */
10346                 fprintf(f,
10347                         "\t\t\t}\n,");
10348         }
10349
10350         /* instr.io - closing curly brace. */
10351         if (n_io || n_io_imm || n_hdrs)
10352                 fprintf(f,
10353                         "\t\t},\n");
10354
10355         /* instr - closing curly brace. */
10356         fprintf(f,
10357                 "\t},\n");
10358 }
10359
10360 static void
10361 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10362 {
10363         fprintf(f,
10364                 "\t{\n"
10365                 "\t\t.type = %s,\n"
10366                 "\t\t.valid = {\n"
10367                 "\t\t\t.header_id = %u,\n"
10368                 "\t\t},\n"
10369                 "\t},\n",
10370                 instr_type_to_name(instr),
10371                 instr->valid.header_id);
10372 }
10373
10374 static void
10375 instr_mov_export(struct instruction *instr, FILE *f)
10376 {
10377         if (instr->type != INSTR_MOV_I)
10378                 fprintf(f,
10379                         "\t{\n"
10380                         "\t\t.type = %s,\n"
10381                         "\t\t.mov = {\n"
10382                         "\t\t\t.dst = {\n"
10383                         "\t\t\t\t.struct_id = %u,\n"
10384                         "\t\t\t\t.n_bits = %u,\n"
10385                         "\t\t\t\t.offset = %u,\n"
10386                         "\t\t\t},\n"
10387                         "\t\t\t.src = {\n"
10388                         "\t\t\t\t.struct_id = %u,\n"
10389                         "\t\t\t\t.n_bits = %u,\n"
10390                         "\t\t\t\t.offset = %u,\n"
10391                         "\t\t\t},\n"
10392                         "\t\t},\n"
10393                         "\t},\n",
10394                         instr_type_to_name(instr),
10395                         instr->mov.dst.struct_id,
10396                         instr->mov.dst.n_bits,
10397                         instr->mov.dst.offset,
10398                         instr->mov.src.struct_id,
10399                         instr->mov.src.n_bits,
10400                         instr->mov.src.offset);
10401         else
10402                 fprintf(f,
10403                         "\t{\n"
10404                         "\t\t.type = %s,\n"
10405                         "\t\t.mov = {\n"
10406                         "\t\t\t.dst = {\n"
10407                         "\t\t\t\t.struct_id = %u,\n"
10408                         "\t\t\t\t.n_bits = %u,\n"
10409                         "\t\t\t\t.offset = %u,\n"
10410                         "\t\t\t}\n,"
10411                         "\t\t\t.src_val = %" PRIu64 ",\n"
10412                         "\t\t},\n"
10413                         "\t},\n",
10414                         instr_type_to_name(instr),
10415                         instr->mov.dst.struct_id,
10416                         instr->mov.dst.n_bits,
10417                         instr->mov.dst.offset,
10418                         instr->mov.src_val);
10419 }
10420
10421 static void
10422 instr_dma_ht_export(struct instruction *instr, FILE *f)
10423 {
10424         uint32_t n_dma = 0, i;
10425
10426         /* n_dma. */
10427         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10428
10429         /* instr. */
10430         fprintf(f,
10431                 "\t{\n"
10432                 "\t\t.type = %s,\n",
10433                 instr_type_to_name(instr));
10434
10435         /* instr.dma. */
10436         fprintf(f,
10437                 "\t\t.dma = {\n");
10438
10439         /* instr.dma.dst. */
10440         fprintf(f,
10441                 "\t\t\t.dst = {\n");
10442
10443         /* instr.dma.dst.header_id. */
10444         fprintf(f,
10445                 "\t\t\t\t.header_id = {");
10446
10447         for (i = 0; i < n_dma; i++)
10448                 fprintf(f,
10449                         "%u, ",
10450                         instr->dma.dst.header_id[i]);
10451
10452         fprintf(f,
10453                 "},\n");
10454
10455         /* instr.dma.dst.struct_id. */
10456         fprintf(f,
10457                 "\t\t\t\t.struct_id = {");
10458
10459         for (i = 0; i < n_dma; i++)
10460                 fprintf(f,
10461                         "%u, ",
10462                         instr->dma.dst.struct_id[i]);
10463
10464         fprintf(f,
10465                 "},\n");
10466
10467         /* instr.dma.dst - closing curly brace. */
10468         fprintf(f,
10469                 "\t\t\t},\n");
10470
10471         /* instr.dma.src. */
10472         fprintf(f,
10473                 "\t\t\t.src = {\n");
10474
10475         /* instr.dma.src.offset. */
10476         fprintf(f,
10477                 "\t\t\t\t.offset = {");
10478
10479         for (i = 0; i < n_dma; i++)
10480                 fprintf(f,
10481                         "%u, ",
10482                         instr->dma.src.offset[i]);
10483
10484         fprintf(f,
10485                 "},\n");
10486
10487         /* instr.dma.src - closing curly brace. */
10488         fprintf(f,
10489                 "\t\t\t},\n");
10490
10491         /* instr.dma.n_bytes. */
10492         fprintf(f,
10493                 "\t\t\t.n_bytes = {");
10494
10495         for (i = 0; i < n_dma; i++)
10496                 fprintf(f,
10497                         "%u, ",
10498                         instr->dma.n_bytes[i]);
10499
10500         fprintf(f,
10501                 "},\n");
10502
10503         /* instr.dma - closing curly brace. */
10504         fprintf(f,
10505                 "\t\t},\n");
10506
10507         /* instr - closing curly brace. */
10508         fprintf(f,
10509                 "\t},\n");
10510 }
10511
10512 static void
10513 instr_alu_export(struct instruction *instr, FILE *f)
10514 {
10515         int imm = 0;
10516
10517         if (instr->type == INSTR_ALU_ADD_MI ||
10518             instr->type == INSTR_ALU_ADD_HI ||
10519             instr->type == INSTR_ALU_SUB_MI ||
10520             instr->type == INSTR_ALU_SUB_HI ||
10521             instr->type == INSTR_ALU_SHL_MI ||
10522             instr->type == INSTR_ALU_SHL_HI ||
10523             instr->type == INSTR_ALU_SHR_MI ||
10524             instr->type == INSTR_ALU_SHR_HI ||
10525             instr->type == INSTR_ALU_AND_I ||
10526             instr->type == INSTR_ALU_OR_I ||
10527             instr->type == INSTR_ALU_XOR_I)
10528                 imm = 1;
10529
10530         if (!imm)
10531                 fprintf(f,
10532                         "\t{\n"
10533                         "\t\t.type = %s,\n"
10534                         "\t\t.alu = {\n"
10535                         "\t\t\t.dst = {\n"
10536                         "\t\t\t\t.struct_id = %u,\n"
10537                         "\t\t\t\t.n_bits = %u,\n"
10538                         "\t\t\t\t.offset = %u,\n"
10539                         "\t\t\t},\n"
10540                         "\t\t\t.src = {\n"
10541                         "\t\t\t\t.struct_id = %u,\n"
10542                         "\t\t\t\t.n_bits = %u,\n"
10543                         "\t\t\t\t.offset = %u,\n"
10544                         "\t\t\t},\n"
10545                         "\t\t},\n"
10546                         "\t},\n",
10547                         instr_type_to_name(instr),
10548                         instr->alu.dst.struct_id,
10549                         instr->alu.dst.n_bits,
10550                         instr->alu.dst.offset,
10551                         instr->alu.src.struct_id,
10552                         instr->alu.src.n_bits,
10553                         instr->alu.src.offset);
10554         else
10555                 fprintf(f,
10556                         "\t{\n"
10557                         "\t\t.type = %s,\n"
10558                         "\t\t.alu = {\n"
10559                         "\t\t\t.dst = {\n"
10560                         "\t\t\t\t.struct_id = %u,\n"
10561                         "\t\t\t\t.n_bits = %u,\n"
10562                         "\t\t\t\t.offset = %u,\n"
10563                         "\t\t\t}\n,"
10564                         "\t\t\t.src_val = %" PRIu64 ",\n"
10565                         "\t\t},\n"
10566                         "\t},\n",
10567                         instr_type_to_name(instr),
10568                         instr->alu.dst.struct_id,
10569                         instr->alu.dst.n_bits,
10570                         instr->alu.dst.offset,
10571                         instr->alu.src_val);
10572 }
10573
10574 static void
10575 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10576 {
10577         int prefetch  = 0, idx_imm = 0, src_imm = 0;
10578
10579         if (instr->type == INSTR_REGPREFETCH_RH ||
10580             instr->type == INSTR_REGPREFETCH_RM ||
10581             instr->type == INSTR_REGPREFETCH_RI)
10582                 prefetch = 1;
10583
10584         /* index is the 3rd operand for the regrd instruction and the 2nd
10585          * operand for the regwr and regadd instructions.
10586          */
10587         if (instr->type == INSTR_REGPREFETCH_RI ||
10588             instr->type == INSTR_REGRD_HRI ||
10589             instr->type == INSTR_REGRD_MRI ||
10590             instr->type == INSTR_REGWR_RIH ||
10591             instr->type == INSTR_REGWR_RIM ||
10592             instr->type == INSTR_REGWR_RII ||
10593             instr->type == INSTR_REGADD_RIH ||
10594             instr->type == INSTR_REGADD_RIM ||
10595             instr->type == INSTR_REGADD_RII)
10596                 idx_imm = 1;
10597
10598         /* src is the 3rd operand for the regwr and regadd instructions. */
10599         if (instr->type == INSTR_REGWR_RHI ||
10600             instr->type == INSTR_REGWR_RMI ||
10601             instr->type == INSTR_REGWR_RII ||
10602             instr->type == INSTR_REGADD_RHI ||
10603             instr->type == INSTR_REGADD_RMI ||
10604             instr->type == INSTR_REGADD_RII)
10605                 src_imm = 1;
10606
10607         /* instr.regarray.regarray_id. */
10608         fprintf(f,
10609                 "\t{\n"
10610                 "\t\t.type = %s,\n"
10611                 "\t\t.regarray = {\n"
10612                 "\t\t\t.regarray_id = %u,\n",
10613                 instr_type_to_name(instr),
10614                 instr->regarray.regarray_id);
10615
10616         /* instr.regarray.idx / instr.regarray.idx_val. */
10617         if (!idx_imm)
10618                 fprintf(f,
10619                         "\t\t\t\t.idx = {\n"
10620                         "\t\t\t\t\t.struct_id = %u,\n"
10621                         "\t\t\t\t\t.n_bits = %u,\n"
10622                         "\t\t\t\t\t.offset = %u,\n"
10623                         "\t\t\t\t},\n",
10624                         instr->regarray.idx.struct_id,
10625                         instr->regarray.idx.n_bits,
10626                         instr->regarray.idx.offset);
10627         else
10628                 fprintf(f,
10629                         "\t\t\t\t.idx_val = %u,\n",
10630                         instr->regarray.idx_val);
10631
10632         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10633         if (!prefetch) {
10634                 if (!src_imm)
10635                         fprintf(f,
10636                                 "\t\t\t\t.dstsrc = {\n"
10637                                 "\t\t\t\t\t.struct_id = %u,\n"
10638                                 "\t\t\t\t\t.n_bits = %u,\n"
10639                                 "\t\t\t\t\t.offset = %u,\n"
10640                                 "\t\t\t\t},\n",
10641                                 instr->regarray.dstsrc.struct_id,
10642                                 instr->regarray.dstsrc.n_bits,
10643                                 instr->regarray.dstsrc.offset);
10644                 else
10645                         fprintf(f,
10646                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10647                                 instr->regarray.dstsrc_val);
10648         }
10649
10650         /* instr.regarray and instr - closing curly braces. */
10651         fprintf(f,
10652                 "\t\t},\n"
10653                 "\t},\n");
10654 }
10655
10656 static void
10657 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10658 {
10659         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
10660
10661         if (instr->type == INSTR_METPREFETCH_H ||
10662             instr->type == INSTR_METPREFETCH_M ||
10663             instr->type == INSTR_METPREFETCH_I)
10664                 prefetch = 1;
10665
10666         /* idx_imm. */
10667         if (instr->type == INSTR_METPREFETCH_I ||
10668             instr->type == INSTR_METER_IHM ||
10669             instr->type == INSTR_METER_IHI ||
10670             instr->type == INSTR_METER_IMM ||
10671             instr->type == INSTR_METER_IMI)
10672                 idx_imm = 1;
10673
10674         /* color_in_imm. */
10675         if (instr->type == INSTR_METER_HHI ||
10676             instr->type == INSTR_METER_HMI ||
10677             instr->type == INSTR_METER_MHI ||
10678             instr->type == INSTR_METER_MMI ||
10679             instr->type == INSTR_METER_IHI ||
10680             instr->type == INSTR_METER_IMI)
10681                 color_in_imm = 1;
10682
10683         /* instr.meter.metarray_id. */
10684         fprintf(f,
10685                 "\t{\n"
10686                 "\t\t.type = %s,\n"
10687                 "\t\t.meter = {\n"
10688                 "\t\t\t.metarray_id = %u,\n",
10689                 instr_type_to_name(instr),
10690                 instr->meter.metarray_id);
10691
10692         /* instr.meter.idx / instr.meter.idx_val. */
10693         if (!idx_imm)
10694                 fprintf(f,
10695                         "\t\t\t.idx = {\n"
10696                         "\t\t\t\t.struct_id = %u,\n"
10697                         "\t\t\t\t.n_bits = %u,\n"
10698                         "\t\t\t\t.offset = %u,\n"
10699                         "\t\t\t},\n",
10700                         instr->meter.idx.struct_id,
10701                         instr->meter.idx.n_bits,
10702                         instr->meter.idx.offset);
10703         else
10704                 fprintf(f,
10705                         "\t\t\t.idx_val = %u,\n",
10706                         instr->meter.idx_val);
10707
10708         if (!prefetch) {
10709                 /* instr.meter.length. */
10710                 fprintf(f,
10711                         "\t\t\t.length = {\n"
10712                         "\t\t\t\t.struct_id = %u,\n"
10713                         "\t\t\t\t.n_bits = %u,\n"
10714                         "\t\t\t\t.offset = %u,\n"
10715                         "\t\t\t},\n",
10716                         instr->meter.length.struct_id,
10717                         instr->meter.length.n_bits,
10718                         instr->meter.length.offset);
10719
10720                 /* instr.meter.color_in / instr.meter.color_in_val. */
10721                 if (!color_in_imm)
10722                         fprintf(f,
10723                                 "\t\t\t.color_in = {\n"
10724                                 "\t\t\t\t.struct_id = %u,\n"
10725                                 "\t\t\t\t.n_bits = %u,\n"
10726                                 "\t\t\t\t.offset = %u,\n"
10727                                 "\t\t\t},\n",
10728                                 instr->meter.color_in.struct_id,
10729                                 instr->meter.color_in.n_bits,
10730                                 instr->meter.color_in.offset);
10731                 else
10732                         fprintf(f,
10733                                 "\t\t\t.color_in_val = %u,\n",
10734                                 (uint32_t)instr->meter.color_in_val);
10735
10736                 /* instr.meter.color_out. */
10737                 fprintf(f,
10738                         "\t\t\t.color_out = {\n"
10739                         "\t\t\t\t.struct_id = %u,\n"
10740                         "\t\t\t\t.n_bits = %u,\n"
10741                         "\t\t\t\t.offset = %u,\n"
10742                         "\t\t\t},\n",
10743                         instr->meter.color_out.struct_id,
10744                         instr->meter.color_out.n_bits,
10745                         instr->meter.color_out.offset);
10746         }
10747
10748         /* instr.meter and instr - closing curly braces. */
10749         fprintf(f,
10750                 "\t\t},\n"
10751                 "\t},\n");
10752 }
10753
10754 static void
10755 instr_table_export(struct instruction *instr,
10756                 FILE *f)
10757 {
10758         fprintf(f,
10759                 "\t{\n"
10760                 "\t\t.type = %s,\n"
10761                 "\t\t.table = {\n"
10762                 "\t\t\t.table_id = %u,\n"
10763                 "\t\t},\n"
10764                 "\t},\n",
10765                 instr_type_to_name(instr),
10766                 instr->table.table_id);
10767 }
10768
10769 static void
10770 instr_learn_export(struct instruction *instr, FILE *f)
10771 {
10772         fprintf(f,
10773                 "\t{\n"
10774                 "\t\t.type = %s,\n"
10775                 "\t\t.learn = {\n"
10776                 "\t\t\t\t.action_id = %u,\n"
10777                 "\t\t},\n"
10778                 "\t},\n",
10779                 instr_type_to_name(instr),
10780                 instr->learn.action_id);
10781 }
10782
10783 static void
10784 instr_forget_export(struct instruction *instr, FILE *f)
10785 {
10786         fprintf(f,
10787                 "\t{\n"
10788                 "\t\t.type = %s,\n"
10789                 "\t},\n",
10790                 instr_type_to_name(instr));
10791 }
10792
10793 static void
10794 instr_extern_export(struct instruction *instr, FILE *f)
10795 {
10796         if (instr->type == INSTR_EXTERN_OBJ)
10797                 fprintf(f,
10798                         "\t{\n"
10799                         "\t\t.type = %s,\n"
10800                         "\t\t.ext_obj = {\n"
10801                         "\t\t\t.ext_obj_id = %u,\n"
10802                         "\t\t\t.func_id = %u,\n"
10803                         "\t\t},\n"
10804                         "\t},\n",
10805                         instr_type_to_name(instr),
10806                         instr->ext_obj.ext_obj_id,
10807                         instr->ext_obj.func_id);
10808         else
10809                 fprintf(f,
10810                         "\t{\n"
10811                         "\t\t.type = %s,\n"
10812                         "\t\t.ext_func = {\n"
10813                         "\t\t\t.ext_func_id = %u,\n"
10814                         "\t\t},\n"
10815                         "\t},\n",
10816                         instr_type_to_name(instr),
10817                         instr->ext_func.ext_func_id);
10818 }
10819
10820 static void
10821 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10822 {
10823         fprintf(f,
10824                 "\t{\n"
10825                 "\t\t.type = %s,\n"
10826                 "\t\t.jmp = {\n"
10827                 "\t\t\t.ip = NULL,\n",
10828                 instr_type_to_name(instr));
10829
10830         switch (instr->type) {
10831         case INSTR_JMP_VALID:
10832         case INSTR_JMP_INVALID:
10833                 fprintf(f,
10834                         "\t\t\t.header_id = %u,\n",
10835                         instr->jmp.header_id);
10836                 break;
10837
10838         case INSTR_JMP_ACTION_HIT:
10839         case INSTR_JMP_ACTION_MISS:
10840                 fprintf(f,
10841                         "\t\t\t.action_id = %u,\n",
10842                         instr->jmp.action_id);
10843                 break;
10844
10845         case INSTR_JMP_EQ:
10846         case INSTR_JMP_EQ_MH:
10847         case INSTR_JMP_EQ_HM:
10848         case INSTR_JMP_EQ_HH:
10849         case INSTR_JMP_NEQ:
10850         case INSTR_JMP_NEQ_MH:
10851         case INSTR_JMP_NEQ_HM:
10852         case INSTR_JMP_NEQ_HH:
10853         case INSTR_JMP_LT:
10854         case INSTR_JMP_LT_MH:
10855         case INSTR_JMP_LT_HM:
10856         case INSTR_JMP_LT_HH:
10857         case INSTR_JMP_GT:
10858         case INSTR_JMP_GT_MH:
10859         case INSTR_JMP_GT_HM:
10860         case INSTR_JMP_GT_HH:
10861                 fprintf(f,
10862                         "\t\t\t.a = {\n"
10863                         "\t\t\t\t.struct_id = %u,\n"
10864                         "\t\t\t\t.n_bits = %u,\n"
10865                         "\t\t\t\t.offset = %u,\n"
10866                         "\t\t\t},\n"
10867                         "\t\t\t.b = {\n"
10868                         "\t\t\t\t.struct_id = %u,\n"
10869                         "\t\t\t\t.n_bits = %u,\n"
10870                         "\t\t\t\t.offset = %u,\n"
10871                         "\t\t\t},\n",
10872                         instr->jmp.a.struct_id,
10873                         instr->jmp.a.n_bits,
10874                         instr->jmp.a.offset,
10875                         instr->jmp.b.struct_id,
10876                         instr->jmp.b.n_bits,
10877                         instr->jmp.b.offset);
10878                 break;
10879
10880         case INSTR_JMP_EQ_I:
10881         case INSTR_JMP_NEQ_I:
10882         case INSTR_JMP_LT_MI:
10883         case INSTR_JMP_LT_HI:
10884         case INSTR_JMP_GT_MI:
10885         case INSTR_JMP_GT_HI:
10886                 fprintf(f,
10887                         "\t\t\t.a = {\n"
10888                         "\t\t\t\t.struct_id = %u,\n"
10889                         "\t\t\t\t.n_bits = %u,\n"
10890                         "\t\t\t\t.offset = %u,\n"
10891                         "\t\t\t}\n,"
10892                         "\t\t\t.b_val = %" PRIu64 ",\n",
10893                         instr->jmp.a.struct_id,
10894                         instr->jmp.a.n_bits,
10895                         instr->jmp.a.offset,
10896                         instr->jmp.b_val);
10897                 break;
10898
10899         default:
10900                 break;
10901         }
10902
10903         fprintf(f,
10904                 "\t\t},\n"
10905                 "\t},\n");
10906 }
10907
10908 static void
10909 instr_return_export(struct instruction *instr,
10910                 FILE *f)
10911 {
10912         fprintf(f,
10913                 "\t{\n"
10914                 "\t\t.type = %s,\n",
10915                 instr_type_to_name(instr));
10916
10917         fprintf(f,
10918                 "\t},\n");
10919 }
10920
10921 static instruction_export_t export_table[] = {
10922         [INSTR_RX] = instr_io_export,
10923
10924         [INSTR_TX] = instr_io_export,
10925         [INSTR_TX_I] = instr_io_export,
10926         [INSTR_DROP] = instr_io_export,
10927
10928         [INSTR_HDR_EXTRACT] = instr_io_export,
10929         [INSTR_HDR_EXTRACT2] = instr_io_export,
10930         [INSTR_HDR_EXTRACT3] = instr_io_export,
10931         [INSTR_HDR_EXTRACT4] = instr_io_export,
10932         [INSTR_HDR_EXTRACT5] = instr_io_export,
10933         [INSTR_HDR_EXTRACT6] = instr_io_export,
10934         [INSTR_HDR_EXTRACT7] = instr_io_export,
10935         [INSTR_HDR_EXTRACT8] = instr_io_export,
10936
10937         [INSTR_HDR_EXTRACT_M] = instr_io_export,
10938
10939         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10940
10941         [INSTR_HDR_EMIT] = instr_io_export,
10942         [INSTR_HDR_EMIT_TX] = instr_io_export,
10943         [INSTR_HDR_EMIT2_TX] = instr_io_export,
10944         [INSTR_HDR_EMIT3_TX] = instr_io_export,
10945         [INSTR_HDR_EMIT4_TX] = instr_io_export,
10946         [INSTR_HDR_EMIT5_TX] = instr_io_export,
10947         [INSTR_HDR_EMIT6_TX] = instr_io_export,
10948         [INSTR_HDR_EMIT7_TX] = instr_io_export,
10949         [INSTR_HDR_EMIT8_TX] = instr_io_export,
10950
10951         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10952         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10953
10954         [INSTR_MOV] = instr_mov_export,
10955         [INSTR_MOV_MH] = instr_mov_export,
10956         [INSTR_MOV_HM] = instr_mov_export,
10957         [INSTR_MOV_HH] = instr_mov_export,
10958         [INSTR_MOV_I] = instr_mov_export,
10959
10960         [INSTR_DMA_HT]  = instr_dma_ht_export,
10961         [INSTR_DMA_HT2] = instr_dma_ht_export,
10962         [INSTR_DMA_HT3] = instr_dma_ht_export,
10963         [INSTR_DMA_HT4] = instr_dma_ht_export,
10964         [INSTR_DMA_HT5] = instr_dma_ht_export,
10965         [INSTR_DMA_HT6] = instr_dma_ht_export,
10966         [INSTR_DMA_HT7] = instr_dma_ht_export,
10967         [INSTR_DMA_HT8] = instr_dma_ht_export,
10968
10969         [INSTR_ALU_ADD] = instr_alu_export,
10970         [INSTR_ALU_ADD_MH] = instr_alu_export,
10971         [INSTR_ALU_ADD_HM] = instr_alu_export,
10972         [INSTR_ALU_ADD_HH] = instr_alu_export,
10973         [INSTR_ALU_ADD_MI] = instr_alu_export,
10974         [INSTR_ALU_ADD_HI] = instr_alu_export,
10975
10976         [INSTR_ALU_SUB] = instr_alu_export,
10977         [INSTR_ALU_SUB_MH] = instr_alu_export,
10978         [INSTR_ALU_SUB_HM] = instr_alu_export,
10979         [INSTR_ALU_SUB_HH] = instr_alu_export,
10980         [INSTR_ALU_SUB_MI] = instr_alu_export,
10981         [INSTR_ALU_SUB_HI] = instr_alu_export,
10982
10983         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10984         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10985         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10986         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10987
10988         [INSTR_ALU_AND] = instr_alu_export,
10989         [INSTR_ALU_AND_MH] = instr_alu_export,
10990         [INSTR_ALU_AND_HM] = instr_alu_export,
10991         [INSTR_ALU_AND_HH] = instr_alu_export,
10992         [INSTR_ALU_AND_I] = instr_alu_export,
10993
10994         [INSTR_ALU_OR] = instr_alu_export,
10995         [INSTR_ALU_OR_MH] = instr_alu_export,
10996         [INSTR_ALU_OR_HM] = instr_alu_export,
10997         [INSTR_ALU_OR_HH] = instr_alu_export,
10998         [INSTR_ALU_OR_I] = instr_alu_export,
10999
11000         [INSTR_ALU_XOR] = instr_alu_export,
11001         [INSTR_ALU_XOR_MH] = instr_alu_export,
11002         [INSTR_ALU_XOR_HM] = instr_alu_export,
11003         [INSTR_ALU_XOR_HH] = instr_alu_export,
11004         [INSTR_ALU_XOR_I] = instr_alu_export,
11005
11006         [INSTR_ALU_SHL] = instr_alu_export,
11007         [INSTR_ALU_SHL_MH] = instr_alu_export,
11008         [INSTR_ALU_SHL_HM] = instr_alu_export,
11009         [INSTR_ALU_SHL_HH] = instr_alu_export,
11010         [INSTR_ALU_SHL_MI] = instr_alu_export,
11011         [INSTR_ALU_SHL_HI] = instr_alu_export,
11012
11013         [INSTR_ALU_SHR] = instr_alu_export,
11014         [INSTR_ALU_SHR_MH] = instr_alu_export,
11015         [INSTR_ALU_SHR_HM] = instr_alu_export,
11016         [INSTR_ALU_SHR_HH] = instr_alu_export,
11017         [INSTR_ALU_SHR_MI] = instr_alu_export,
11018         [INSTR_ALU_SHR_HI] = instr_alu_export,
11019
11020         [INSTR_REGPREFETCH_RH] = instr_reg_export,
11021         [INSTR_REGPREFETCH_RM] = instr_reg_export,
11022         [INSTR_REGPREFETCH_RI] = instr_reg_export,
11023
11024         [INSTR_REGRD_HRH] = instr_reg_export,
11025         [INSTR_REGRD_HRM] = instr_reg_export,
11026         [INSTR_REGRD_MRH] = instr_reg_export,
11027         [INSTR_REGRD_MRM] = instr_reg_export,
11028         [INSTR_REGRD_HRI] = instr_reg_export,
11029         [INSTR_REGRD_MRI] = instr_reg_export,
11030
11031         [INSTR_REGWR_RHH] = instr_reg_export,
11032         [INSTR_REGWR_RHM] = instr_reg_export,
11033         [INSTR_REGWR_RMH] = instr_reg_export,
11034         [INSTR_REGWR_RMM] = instr_reg_export,
11035         [INSTR_REGWR_RHI] = instr_reg_export,
11036         [INSTR_REGWR_RMI] = instr_reg_export,
11037         [INSTR_REGWR_RIH] = instr_reg_export,
11038         [INSTR_REGWR_RIM] = instr_reg_export,
11039         [INSTR_REGWR_RII] = instr_reg_export,
11040
11041         [INSTR_REGADD_RHH] = instr_reg_export,
11042         [INSTR_REGADD_RHM] = instr_reg_export,
11043         [INSTR_REGADD_RMH] = instr_reg_export,
11044         [INSTR_REGADD_RMM] = instr_reg_export,
11045         [INSTR_REGADD_RHI] = instr_reg_export,
11046         [INSTR_REGADD_RMI] = instr_reg_export,
11047         [INSTR_REGADD_RIH] = instr_reg_export,
11048         [INSTR_REGADD_RIM] = instr_reg_export,
11049         [INSTR_REGADD_RII] = instr_reg_export,
11050
11051         [INSTR_METPREFETCH_H] = instr_meter_export,
11052         [INSTR_METPREFETCH_M] = instr_meter_export,
11053         [INSTR_METPREFETCH_I] = instr_meter_export,
11054
11055         [INSTR_METER_HHM] = instr_meter_export,
11056         [INSTR_METER_HHI] = instr_meter_export,
11057         [INSTR_METER_HMM] = instr_meter_export,
11058         [INSTR_METER_HMI] = instr_meter_export,
11059         [INSTR_METER_MHM] = instr_meter_export,
11060         [INSTR_METER_MHI] = instr_meter_export,
11061         [INSTR_METER_MMM] = instr_meter_export,
11062         [INSTR_METER_MMI] = instr_meter_export,
11063         [INSTR_METER_IHM] = instr_meter_export,
11064         [INSTR_METER_IHI] = instr_meter_export,
11065         [INSTR_METER_IMM] = instr_meter_export,
11066         [INSTR_METER_IMI] = instr_meter_export,
11067
11068         [INSTR_TABLE] = instr_table_export,
11069         [INSTR_TABLE_AF] = instr_table_export,
11070         [INSTR_SELECTOR] = instr_table_export,
11071         [INSTR_LEARNER] = instr_table_export,
11072         [INSTR_LEARNER_AF] = instr_table_export,
11073
11074         [INSTR_LEARNER_LEARN] = instr_learn_export,
11075         [INSTR_LEARNER_FORGET] = instr_forget_export,
11076
11077         [INSTR_EXTERN_OBJ] = instr_extern_export,
11078         [INSTR_EXTERN_FUNC] = instr_extern_export,
11079
11080         [INSTR_JMP] = instr_jmp_export,
11081         [INSTR_JMP_VALID] = instr_jmp_export,
11082         [INSTR_JMP_INVALID] = instr_jmp_export,
11083         [INSTR_JMP_HIT] = instr_jmp_export,
11084         [INSTR_JMP_MISS] = instr_jmp_export,
11085         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11086         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11087
11088         [INSTR_JMP_EQ] = instr_jmp_export,
11089         [INSTR_JMP_EQ_MH] = instr_jmp_export,
11090         [INSTR_JMP_EQ_HM] = instr_jmp_export,
11091         [INSTR_JMP_EQ_HH] = instr_jmp_export,
11092         [INSTR_JMP_EQ_I] = instr_jmp_export,
11093
11094         [INSTR_JMP_NEQ] = instr_jmp_export,
11095         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11096         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11097         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11098         [INSTR_JMP_NEQ_I] = instr_jmp_export,
11099
11100         [INSTR_JMP_LT] = instr_jmp_export,
11101         [INSTR_JMP_LT_MH] = instr_jmp_export,
11102         [INSTR_JMP_LT_HM] = instr_jmp_export,
11103         [INSTR_JMP_LT_HH] = instr_jmp_export,
11104         [INSTR_JMP_LT_MI] = instr_jmp_export,
11105         [INSTR_JMP_LT_HI] = instr_jmp_export,
11106
11107         [INSTR_JMP_GT] = instr_jmp_export,
11108         [INSTR_JMP_GT_MH] = instr_jmp_export,
11109         [INSTR_JMP_GT_HM] = instr_jmp_export,
11110         [INSTR_JMP_GT_HH] = instr_jmp_export,
11111         [INSTR_JMP_GT_MI] = instr_jmp_export,
11112         [INSTR_JMP_GT_HI] = instr_jmp_export,
11113
11114         [INSTR_RETURN] = instr_return_export,
11115 };
11116
11117 static void
11118 action_data_codegen(struct action *a, FILE *f)
11119 {
11120         uint32_t i;
11121
11122         fprintf(f,
11123                 "static const struct instruction action_%s_instructions[] = {\n",
11124                 a->name);
11125
11126         for (i = 0; i < a->n_instructions; i++) {
11127                 struct instruction *instr = &a->instructions[i];
11128                 instruction_export_t func = export_table[instr->type];
11129
11130                 func(instr, f);
11131         }
11132
11133         fprintf(f, "};\n");
11134 }
11135
11136 static const char *
11137 instr_type_to_func(struct instruction *instr)
11138 {
11139         switch (instr->type) {
11140         case INSTR_RX: return NULL;
11141
11142         case INSTR_TX: return "__instr_tx_exec";
11143         case INSTR_TX_I: return "__instr_tx_i_exec";
11144         case INSTR_DROP: return "__instr_drop_exec";
11145
11146         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11147         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11148         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11149         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11150         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11151         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11152         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11153         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11154
11155         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11156
11157         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11158
11159         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11160         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11161         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11162         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11163         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11164         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11165         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11166         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11167         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11168
11169         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11170         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11171
11172         case INSTR_MOV: return "__instr_mov_exec";
11173         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11174         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11175         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11176         case INSTR_MOV_I: return "__instr_mov_i_exec";
11177
11178         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11179         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11180         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11181         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11182         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11183         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11184         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11185         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11186
11187         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11188         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11189         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11190         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11191         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11192         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11193
11194         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11195         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11196         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11197         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11198         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11199         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11200
11201         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11202         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11203         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11204         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11205
11206         case INSTR_ALU_AND: return "__instr_alu_and_exec";
11207         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11208         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11209         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11210         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11211
11212         case INSTR_ALU_OR: return "__instr_alu_or_exec";
11213         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11214         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11215         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11216         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11217
11218         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11219         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11220         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11221         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11222         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11223
11224         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11225         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11226         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11227         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11228         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11229         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11230
11231         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11232         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11233         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11234         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11235         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11236         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11237
11238         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11239         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11240         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11241
11242         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11243         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11244         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11245         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11246         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11247         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11248
11249         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11250         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11251         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11252         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11253         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11254         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11255         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11256         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11257         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11258
11259         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11260         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11261         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11262         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11263         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11264         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11265         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11266         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11267         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11268
11269         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11270         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11271         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11272
11273         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11274         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11275         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11276         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11277         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11278         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11279         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11280         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11281         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11282         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11283         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11284         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11285
11286         case INSTR_TABLE: return NULL;
11287         case INSTR_TABLE_AF: return NULL;
11288         case INSTR_SELECTOR: return NULL;
11289         case INSTR_LEARNER: return NULL;
11290         case INSTR_LEARNER_AF: return NULL;
11291
11292         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11293         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11294
11295         case INSTR_EXTERN_OBJ: return NULL;
11296         case INSTR_EXTERN_FUNC: return NULL;
11297
11298         case INSTR_JMP: return NULL;
11299         case INSTR_JMP_VALID: return NULL;
11300         case INSTR_JMP_INVALID: return NULL;
11301         case INSTR_JMP_HIT: return NULL;
11302         case INSTR_JMP_MISS: return NULL;
11303         case INSTR_JMP_ACTION_HIT: return NULL;
11304         case INSTR_JMP_ACTION_MISS: return NULL;
11305         case INSTR_JMP_EQ: return NULL;
11306         case INSTR_JMP_EQ_MH: return NULL;
11307         case INSTR_JMP_EQ_HM: return NULL;
11308         case INSTR_JMP_EQ_HH: return NULL;
11309         case INSTR_JMP_EQ_I: return NULL;
11310         case INSTR_JMP_NEQ: return NULL;
11311         case INSTR_JMP_NEQ_MH: return NULL;
11312         case INSTR_JMP_NEQ_HM: return NULL;
11313         case INSTR_JMP_NEQ_HH: return NULL;
11314         case INSTR_JMP_NEQ_I: return NULL;
11315         case INSTR_JMP_LT: return NULL;
11316         case INSTR_JMP_LT_MH: return NULL;
11317         case INSTR_JMP_LT_HM: return NULL;
11318         case INSTR_JMP_LT_HH: return NULL;
11319         case INSTR_JMP_LT_MI: return NULL;
11320         case INSTR_JMP_LT_HI: return NULL;
11321         case INSTR_JMP_GT: return NULL;
11322         case INSTR_JMP_GT_MH: return NULL;
11323         case INSTR_JMP_GT_HM: return NULL;
11324         case INSTR_JMP_GT_HH: return NULL;
11325         case INSTR_JMP_GT_MI: return NULL;
11326         case INSTR_JMP_GT_HI: return NULL;
11327
11328         case INSTR_RETURN: return NULL;
11329
11330         default: return NULL;
11331         }
11332 }
11333
11334 static void
11335 action_instr_does_tx_codegen(struct action *a,
11336                         uint32_t instr_pos,
11337                         struct instruction *instr,
11338                         FILE *f)
11339 {
11340         fprintf(f,
11341                 "%s(p, t, &action_%s_instructions[%u]);\n"
11342                 "\tthread_ip_reset(p, t);\n"
11343                 "\tinstr_rx_exec(p);\n"
11344                 "\treturn;\n",
11345                 instr_type_to_func(instr),
11346                 a->name,
11347                 instr_pos);
11348 }
11349
11350 static void
11351 action_instr_extern_obj_codegen(struct action *a,
11352                                 uint32_t instr_pos,
11353                                 FILE *f)
11354 {
11355         fprintf(f,
11356                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11357                 a->name,
11358                 instr_pos);
11359 }
11360
11361 static void
11362 action_instr_extern_func_codegen(struct action *a,
11363                                  uint32_t instr_pos,
11364                                  FILE *f)
11365 {
11366         fprintf(f,
11367                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11368                 a->name,
11369                 instr_pos);
11370 }
11371
11372 static void
11373 action_instr_jmp_codegen(struct action *a,
11374                          uint32_t instr_pos,
11375                          struct instruction *instr,
11376                          struct instruction_data *data,
11377                          FILE *f)
11378 {
11379         switch (instr->type) {
11380         case INSTR_JMP:
11381                 fprintf(f,
11382                         "goto %s;\n",
11383                         data->jmp_label);
11384                 return;
11385
11386         case INSTR_JMP_VALID:
11387                 fprintf(f,
11388                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11389                         "\t\tgoto %s;\n",
11390                         a->name,
11391                         instr_pos,
11392                         data->jmp_label);
11393                 return;
11394
11395         case INSTR_JMP_INVALID:
11396                 fprintf(f,
11397                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11398                         "\t\tgoto %s;\n",
11399                         a->name,
11400                         instr_pos,
11401                         data->jmp_label);
11402                 return;
11403
11404         case INSTR_JMP_HIT:
11405                 fprintf(f,
11406                         "if (t->hit)\n"
11407                         "\t\tgoto %s;\n",
11408                         data->jmp_label);
11409                 return;
11410
11411         case INSTR_JMP_MISS:
11412                 fprintf(f,
11413                         "if (!t->hit)\n"
11414                         "\t\tgoto %s;\n",
11415                         data->jmp_label);
11416                 return;
11417
11418         case INSTR_JMP_ACTION_HIT:
11419                 fprintf(f,
11420                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11421                         "\t\tgoto %s;\n",
11422                         a->name,
11423                         instr_pos,
11424                         data->jmp_label);
11425                 return;
11426
11427         case INSTR_JMP_ACTION_MISS:
11428                 fprintf(f,
11429                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11430                         "\t\tgoto %s;\n",
11431                         a->name,
11432                         instr_pos,
11433                         data->jmp_label);
11434                 return;
11435
11436         case INSTR_JMP_EQ:
11437                 fprintf(f,
11438                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11439                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11440                         "\t\tgoto %s;\n",
11441                         a->name,
11442                         instr_pos,
11443                         a->name,
11444                         instr_pos,
11445                         data->jmp_label);
11446                 return;
11447
11448         case INSTR_JMP_EQ_MH:
11449                 fprintf(f,
11450                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11451                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11452                         "\t\tgoto %s;\n",
11453                         a->name,
11454                         instr_pos,
11455                         a->name,
11456                         instr_pos,
11457                         data->jmp_label);
11458                 return;
11459
11460         case INSTR_JMP_EQ_HM:
11461                 fprintf(f,
11462                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11463                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11464                         "\t\tgoto %s;\n",
11465                         a->name,
11466                         instr_pos,
11467                         a->name,
11468                         instr_pos,
11469                         data->jmp_label);
11470                 return;
11471
11472         case INSTR_JMP_EQ_HH:
11473                 fprintf(f,
11474                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11475                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11476                         "\t\tgoto %s;\n",
11477                         a->name,
11478                         instr_pos,
11479                         a->name,
11480                         instr_pos,
11481                         data->jmp_label);
11482                 return;
11483
11484         case INSTR_JMP_EQ_I:
11485                 fprintf(f,
11486                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11487                         "action_%s_instructions[%u].jmp.b_val)\n"
11488                         "\t\tgoto %s;\n",
11489                         a->name,
11490                         instr_pos,
11491                         a->name,
11492                         instr_pos,
11493                         data->jmp_label);
11494                 return;
11495
11496         case INSTR_JMP_NEQ:
11497                 fprintf(f,
11498                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11499                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11500                         "\t\tgoto %s;\n",
11501                         a->name,
11502                         instr_pos,
11503                         a->name,
11504                         instr_pos,
11505                         data->jmp_label);
11506                 return;
11507
11508         case INSTR_JMP_NEQ_MH:
11509                 fprintf(f,
11510                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11511                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11512                         "\t\tgoto %s;\n",
11513                         a->name,
11514                         instr_pos,
11515                         a->name,
11516                         instr_pos,
11517                         data->jmp_label);
11518                 return;
11519
11520         case INSTR_JMP_NEQ_HM:
11521                 fprintf(f,
11522                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11523                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11524                         "\t\tgoto %s;\n",
11525                         a->name,
11526                         instr_pos,
11527                         a->name,
11528                         instr_pos,
11529                         data->jmp_label);
11530                 return;
11531
11532         case INSTR_JMP_NEQ_HH:
11533                 fprintf(f,
11534                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11535                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11536                         "\t\tgoto %s;\n",
11537                         a->name,
11538                         instr_pos,
11539                         a->name,
11540                         instr_pos,
11541                         data->jmp_label);
11542                 return;
11543
11544         case INSTR_JMP_NEQ_I:
11545                 fprintf(f,
11546                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11547                         "action_%s_instructions[%u].jmp.b_val)\n"
11548                         "\t\tgoto %s;\n",
11549                         a->name,
11550                         instr_pos,
11551                         a->name,
11552                         instr_pos,
11553                         data->jmp_label);
11554                 return;
11555
11556         case INSTR_JMP_LT:
11557                 fprintf(f,
11558                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11559                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11560                         "\t\tgoto %s;\n",
11561                         a->name,
11562                         instr_pos,
11563                         a->name,
11564                         instr_pos,
11565                         data->jmp_label);
11566                 return;
11567
11568         case INSTR_JMP_LT_MH:
11569                 fprintf(f,
11570                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11571                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11572                         "\t\tgoto %s;\n",
11573                         a->name,
11574                         instr_pos,
11575                         a->name,
11576                         instr_pos,
11577                         data->jmp_label);
11578                 return;
11579
11580         case INSTR_JMP_LT_HM:
11581                 fprintf(f,
11582                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11583                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11584                         "\t\tgoto %s;\n",
11585                         a->name,
11586                         instr_pos,
11587                         a->name,
11588                         instr_pos,
11589                         data->jmp_label);
11590                 return;
11591
11592         case INSTR_JMP_LT_HH:
11593                 fprintf(f,
11594                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11595                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11596                         "\t\tgoto %s;\n",
11597                         a->name,
11598                         instr_pos,
11599                         a->name,
11600                         instr_pos,
11601                         data->jmp_label);
11602                 return;
11603
11604         case INSTR_JMP_LT_MI:
11605                 fprintf(f,
11606                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11607                         "action_%s_instructions[%u].jmp.b_val)\n"
11608                         "\t\tgoto %s;\n",
11609                         a->name,
11610                         instr_pos,
11611                         a->name,
11612                         instr_pos,
11613                         data->jmp_label);
11614                 return;
11615
11616         case INSTR_JMP_LT_HI:
11617                 fprintf(f,
11618                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11619                         "action_%s_instructions[%u].jmp.b_val)\n"
11620                         "\t\tgoto %s;\n",
11621                         a->name,
11622                         instr_pos,
11623                         a->name,
11624                         instr_pos,
11625                         data->jmp_label);
11626                 return;
11627
11628         case INSTR_JMP_GT:
11629                 fprintf(f,
11630                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11631                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11632                         "\t\tgoto %s;\n",
11633                         a->name,
11634                         instr_pos,
11635                         a->name,
11636                         instr_pos,
11637                         data->jmp_label);
11638                 return;
11639
11640         case INSTR_JMP_GT_MH:
11641                 fprintf(f,
11642                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11643                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11644                         "\t\tgoto %s;\n",
11645                         a->name,
11646                         instr_pos,
11647                         a->name,
11648                         instr_pos,
11649                         data->jmp_label);
11650                 return;
11651
11652         case INSTR_JMP_GT_HM:
11653                 fprintf(f,
11654                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11655                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11656                         "\t\tgoto %s;\n",
11657                         a->name,
11658                         instr_pos,
11659                         a->name,
11660                         instr_pos,
11661                         data->jmp_label);
11662                 return;
11663
11664         case INSTR_JMP_GT_HH:
11665                 fprintf(f,
11666                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11667                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11668                         "\t\tgoto %s;\n",
11669                         a->name,
11670                         instr_pos,
11671                         a->name,
11672                         instr_pos,
11673                         data->jmp_label);
11674                 return;
11675
11676         case INSTR_JMP_GT_MI:
11677                 fprintf(f,
11678                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11679                         "action_%s_instructions[%u].jmp.b_val)\n"
11680                         "\t\tgoto %s;\n",
11681                         a->name,
11682                         instr_pos,
11683                         a->name,
11684                         instr_pos,
11685                         data->jmp_label);
11686                 return;
11687
11688         case INSTR_JMP_GT_HI:
11689                 fprintf(f,
11690                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11691                         "action_%s_instructions[%u].jmp.b_val)\n"
11692                         "\t\tgoto %s;\n",
11693                         a->name,
11694                         instr_pos,
11695                         a->name,
11696                         instr_pos,
11697                         data->jmp_label);
11698                 return;
11699
11700         default:
11701                 return;
11702         }
11703 }
11704
11705 static void
11706 action_instr_return_codegen(FILE *f)
11707 {
11708         fprintf(f,
11709                 "return;\n");
11710 }
11711
11712 static void
11713 action_instr_codegen(struct action *a, FILE *f)
11714 {
11715         uint32_t i;
11716
11717         fprintf(f,
11718                 "void\n"
11719                 "action_%s_run(struct rte_swx_pipeline *p)\n"
11720                 "{\n"
11721                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11722                 "\n",
11723                 a->name);
11724
11725         for (i = 0; i < a->n_instructions; i++) {
11726                 struct instruction *instr = &a->instructions[i];
11727                 struct instruction_data *data = &a->instruction_data[i];
11728
11729                 /* Label, if present. */
11730                 if (data->label[0])
11731                         fprintf(f, "\n%s : ", data->label);
11732                 else
11733                         fprintf(f, "\n\t");
11734
11735                 /* TX instruction type. */
11736                 if (instruction_does_tx(instr)) {
11737                         action_instr_does_tx_codegen(a, i, instr, f);
11738                         continue;
11739                 }
11740
11741                 /* Extern object/function instruction type. */
11742                 if (instr->type == INSTR_EXTERN_OBJ) {
11743                         action_instr_extern_obj_codegen(a, i, f);
11744                         continue;
11745                 }
11746
11747                 if (instr->type == INSTR_EXTERN_FUNC) {
11748                         action_instr_extern_func_codegen(a, i, f);
11749                         continue;
11750                 }
11751
11752                 /* Jump instruction type. */
11753                 if (instruction_is_jmp(instr)) {
11754                         action_instr_jmp_codegen(a, i, instr, data, f);
11755                         continue;
11756                 }
11757
11758                 /* Return instruction type. */
11759                 if (instr->type == INSTR_RETURN) {
11760                         action_instr_return_codegen(f);
11761                         continue;
11762                 }
11763
11764                 /* Any other instruction type. */
11765                 fprintf(f,
11766                         "%s(p, t, &action_%s_instructions[%u]);\n",
11767                         instr_type_to_func(instr),
11768                         a->name,
11769                         i);
11770         }
11771
11772         fprintf(f, "}\n\n");
11773 }
11774
11775 struct instruction_group {
11776         TAILQ_ENTRY(instruction_group) node;
11777
11778         uint32_t group_id;
11779
11780         uint32_t first_instr_id;
11781
11782         uint32_t last_instr_id;
11783
11784         instr_exec_t func;
11785 };
11786
11787 TAILQ_HEAD(instruction_group_list, instruction_group);
11788
11789 static struct instruction_group *
11790 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11791 {
11792         struct instruction_group *g;
11793
11794         TAILQ_FOREACH(g, igl, node)
11795                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11796                         return g;
11797
11798         return NULL;
11799 }
11800
11801 static void
11802 instruction_group_list_free(struct instruction_group_list *igl)
11803 {
11804         if (!igl)
11805                 return;
11806
11807         for ( ; ; ) {
11808                 struct instruction_group *g;
11809
11810                 g = TAILQ_FIRST(igl);
11811                 if (!g)
11812                         break;
11813
11814                 TAILQ_REMOVE(igl, g, node);
11815                 free(g);
11816         }
11817
11818         free(igl);
11819 }
11820
11821 static struct instruction_group_list *
11822 instruction_group_list_create(struct rte_swx_pipeline *p)
11823 {
11824         struct instruction_group_list *igl = NULL;
11825         struct instruction_group *g = NULL;
11826         uint32_t n_groups = 0, i;
11827
11828         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11829                 goto error;
11830
11831         /* List init. */
11832         igl = calloc(1, sizeof(struct instruction_group_list));
11833         if (!igl)
11834                 goto error;
11835
11836         TAILQ_INIT(igl);
11837
11838         /* Allocate the first group. */
11839         g = calloc(1, sizeof(struct instruction_group));
11840         if (!g)
11841                 goto error;
11842
11843         /* Iteration 1: Separate the instructions into groups based on the thread yield
11844          * instructions. Do not worry about the jump instructions at this point.
11845          */
11846         for (i = 0; i < p->n_instructions; i++) {
11847                 struct instruction *instr = &p->instructions[i];
11848
11849                 /* Check for thread yield instructions. */
11850                 if (!instruction_does_thread_yield(instr))
11851                         continue;
11852
11853                 /* If the current group contains at least one instruction, then finalize it (with
11854                  * the previous instruction), add it to the list and allocate a new group (that
11855                  * starts with the current instruction).
11856                  */
11857                 if (i - g->first_instr_id) {
11858                         /* Finalize the group. */
11859                         g->last_instr_id = i - 1;
11860
11861                         /* Add the group to the list. Advance the number of groups. */
11862                         TAILQ_INSERT_TAIL(igl, g, node);
11863                         n_groups++;
11864
11865                         /* Allocate a new group. */
11866                         g = calloc(1, sizeof(struct instruction_group));
11867                         if (!g)
11868                                 goto error;
11869
11870                         /* Initialize the new group. */
11871                         g->group_id = n_groups;
11872                         g->first_instr_id = i;
11873                 }
11874
11875                 /* Finalize the current group (with the current instruction, therefore this group
11876                  * contains just the current thread yield instruction), add it to the list and
11877                  * allocate a new group (that starts with the next instruction).
11878                  */
11879
11880                 /* Finalize the group. */
11881                 g->last_instr_id = i;
11882
11883                 /* Add the group to the list. Advance the number of groups. */
11884                 TAILQ_INSERT_TAIL(igl, g, node);
11885                 n_groups++;
11886
11887                 /* Allocate a new group. */
11888                 g = calloc(1, sizeof(struct instruction_group));
11889                 if (!g)
11890                         goto error;
11891
11892                 /* Initialize the new group. */
11893                 g->group_id = n_groups;
11894                 g->first_instr_id = i + 1;
11895         }
11896
11897         /* Handle the last group. */
11898         if (i - g->first_instr_id) {
11899                 /* Finalize the group. */
11900                 g->last_instr_id = i - 1;
11901
11902                 /* Add the group to the list. Advance the number of groups. */
11903                 TAILQ_INSERT_TAIL(igl, g, node);
11904                 n_groups++;
11905         } else
11906                 free(g);
11907
11908         g = NULL;
11909
11910         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11911          * the destination of a jump instruction located in a different group ("far jump"), then the
11912          * current group has to be split, so that the instruction representing the far jump
11913          * destination is at the start of its group.
11914          */
11915         for ( ; ; ) {
11916                 int is_modified = 0;
11917
11918                 for (i = 0; i < p->n_instructions; i++) {
11919                         struct instruction_data *data = &p->instruction_data[i];
11920                         struct instruction_group *g;
11921                         uint32_t j;
11922
11923                         /* Continue when the current instruction is not a jump destination. */
11924                         if (!data->n_users)
11925                                 continue;
11926
11927                         g = instruction_group_list_group_find(igl, i);
11928                         if (!g)
11929                                 goto error;
11930
11931                         /* Find out all the jump instructions with this destination. */
11932                         for (j = 0; j < p->n_instructions; j++) {
11933                                 struct instruction *jmp_instr = &p->instructions[j];
11934                                 struct instruction_data *jmp_data = &p->instruction_data[j];
11935                                 struct instruction_group *jmp_g, *new_g;
11936
11937                                 /* Continue when not a jump instruction. Even when jump instruction,
11938                                  * continue when the jump destination is not this instruction.
11939                                  */
11940                                 if (!instruction_is_jmp(jmp_instr) ||
11941                                     strcmp(jmp_data->jmp_label, data->label))
11942                                         continue;
11943
11944                                 jmp_g = instruction_group_list_group_find(igl, j);
11945                                 if (!jmp_g)
11946                                         goto error;
11947
11948                                 /* Continue when both the jump instruction and the jump destination
11949                                  * instruction are in the same group. Even when in different groups,
11950                                  * still continue if the jump destination instruction is already the
11951                                  * first instruction of its group.
11952                                  */
11953                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11954                                         continue;
11955
11956                                 /* Split the group of the current jump destination instruction to
11957                                  * make this instruction the first instruction of a new group.
11958                                  */
11959                                 new_g = calloc(1, sizeof(struct instruction_group));
11960                                 if (!new_g)
11961                                         goto error;
11962
11963                                 new_g->group_id = n_groups;
11964                                 new_g->first_instr_id = i;
11965                                 new_g->last_instr_id = g->last_instr_id;
11966
11967                                 g->last_instr_id = i - 1;
11968
11969                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11970                                 n_groups++;
11971                                 is_modified = 1;
11972
11973                                 /* The decision to split this group (to make the current instruction
11974                                  * the first instruction of a new group) is already taken and fully
11975                                  * implemented, so no need to search for more reasons to do it.
11976                                  */
11977                                 break;
11978                         }
11979                 }
11980
11981                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11982                  * previously considered local (i.e. the jump destination is in the same group as
11983                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11984                  * different group than the jump instruction). Wost case scenario: each instruction
11985                  * that is a jump destination ends up as the first instruction of its group.
11986                  */
11987                 if (!is_modified)
11988                         break;
11989         }
11990
11991         /* Re-assign the group IDs to be in incremental order. */
11992         i = 0;
11993         TAILQ_FOREACH(g, igl, node) {
11994                 g->group_id = i;
11995
11996                 i++;
11997         }
11998
11999         return igl;
12000
12001 error:
12002         instruction_group_list_free(igl);
12003
12004         free(g);
12005
12006         return NULL;
12007 }
12008
12009 static void
12010 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12011                                uint32_t instr_pos,
12012                                struct instruction *instr,
12013                                FILE *f)
12014 {
12015         fprintf(f,
12016                 "%s(p, t, &pipeline_instructions[%u]);\n"
12017                 "\tthread_ip_reset(p, t);\n"
12018                 "\tinstr_rx_exec(p);\n"
12019                 "\treturn;\n",
12020                 instr_type_to_func(instr),
12021                 instr_pos);
12022 }
12023
12024 static int
12025 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12026                            struct instruction_group_list *igl,
12027                            uint32_t jmp_instr_id,
12028                            struct instruction *jmp_instr,
12029                            struct instruction_data *jmp_data,
12030                            FILE *f)
12031 {
12032         struct instruction_group *jmp_g, *g;
12033         struct instruction_data *data;
12034         uint32_t instr_id;
12035
12036         switch (jmp_instr->type) {
12037         case INSTR_JMP:
12038                 break;
12039
12040         case INSTR_JMP_VALID:
12041                 fprintf(f,
12042                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12043                         jmp_instr_id);
12044                 break;
12045
12046         case INSTR_JMP_INVALID:
12047                 fprintf(f,
12048                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12049                         jmp_instr_id);
12050                 break;
12051
12052         case INSTR_JMP_HIT:
12053                 fprintf(f,
12054                         "if (t->hit)\n");
12055                 break;
12056
12057         case INSTR_JMP_MISS:
12058                 fprintf(f,
12059                         "if (!t->hit)\n");
12060                 break;
12061
12062         case INSTR_JMP_ACTION_HIT:
12063                 fprintf(f,
12064                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12065                         jmp_instr_id);
12066                 break;
12067
12068         case INSTR_JMP_ACTION_MISS:
12069                 fprintf(f,
12070                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12071                         jmp_instr_id);
12072                 break;
12073
12074         case INSTR_JMP_EQ:
12075                 fprintf(f,
12076                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12077                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12078                         jmp_instr_id,
12079                         jmp_instr_id);
12080                 break;
12081
12082         case INSTR_JMP_EQ_MH:
12083                 fprintf(f,
12084                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12085                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12086                         jmp_instr_id,
12087                         jmp_instr_id);
12088                 break;
12089
12090         case INSTR_JMP_EQ_HM:
12091                 fprintf(f,
12092                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12093                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12094                         jmp_instr_id,
12095                         jmp_instr_id);
12096                 break;
12097
12098         case INSTR_JMP_EQ_HH:
12099                 fprintf(f,
12100                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12101                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12102                         jmp_instr_id,
12103                         jmp_instr_id);
12104                 break;
12105
12106         case INSTR_JMP_EQ_I:
12107                 fprintf(f,
12108                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12109                         "pipeline_instructions[%u].jmp.b_val)",
12110                         jmp_instr_id,
12111                         jmp_instr_id);
12112                 break;
12113
12114         case INSTR_JMP_NEQ:
12115                 fprintf(f,
12116                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12117                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12118                         jmp_instr_id,
12119                         jmp_instr_id);
12120                 break;
12121
12122         case INSTR_JMP_NEQ_MH:
12123                 fprintf(f,
12124                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12125                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12126                         jmp_instr_id,
12127                         jmp_instr_id);
12128                 break;
12129
12130         case INSTR_JMP_NEQ_HM:
12131                 fprintf(f,
12132                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12133                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12134                         jmp_instr_id,
12135                         jmp_instr_id);
12136                 break;
12137
12138         case INSTR_JMP_NEQ_HH:
12139                 fprintf(f,
12140                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12141                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12142                         jmp_instr_id,
12143                         jmp_instr_id);
12144                 break;
12145
12146         case INSTR_JMP_NEQ_I:
12147                 fprintf(f,
12148                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12149                         "pipeline_instructions[%u].jmp.b_val)",
12150                         jmp_instr_id,
12151                         jmp_instr_id);
12152                 break;
12153
12154         case INSTR_JMP_LT:
12155                 fprintf(f,
12156                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12157                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12158                         jmp_instr_id,
12159                         jmp_instr_id);
12160                 break;
12161
12162         case INSTR_JMP_LT_MH:
12163                 fprintf(f,
12164                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12165                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12166                         jmp_instr_id,
12167                         jmp_instr_id);
12168                 break;
12169
12170         case INSTR_JMP_LT_HM:
12171                 fprintf(f,
12172                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12173                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12174                         jmp_instr_id,
12175                         jmp_instr_id);
12176                 break;
12177
12178         case INSTR_JMP_LT_HH:
12179                 fprintf(f,
12180                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12181                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12182                         jmp_instr_id,
12183                         jmp_instr_id);
12184                 break;
12185
12186         case INSTR_JMP_LT_MI:
12187                 fprintf(f,
12188                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12189                         "pipeline_instructions[%u].jmp.b_val)",
12190                         jmp_instr_id,
12191                         jmp_instr_id);
12192                 break;
12193
12194         case INSTR_JMP_LT_HI:
12195                 fprintf(f,
12196                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12197                         "pipeline_instructions[%u].jmp.b_val)",
12198                         jmp_instr_id,
12199                         jmp_instr_id);
12200                 break;
12201
12202         case INSTR_JMP_GT:
12203                 fprintf(f,
12204                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12205                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12206                         jmp_instr_id,
12207                         jmp_instr_id);
12208                 break;
12209
12210         case INSTR_JMP_GT_MH:
12211                 fprintf(f,
12212                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12213                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12214                         jmp_instr_id,
12215                         jmp_instr_id);
12216                 break;
12217
12218         case INSTR_JMP_GT_HM:
12219                 fprintf(f,
12220                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12221                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12222                         jmp_instr_id,
12223                         jmp_instr_id);
12224                 break;
12225
12226         case INSTR_JMP_GT_HH:
12227                 fprintf(f,
12228                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12229                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12230                         jmp_instr_id,
12231                         jmp_instr_id);
12232                 break;
12233
12234         case INSTR_JMP_GT_MI:
12235                 fprintf(f,
12236                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12237                         "pipeline_instructions[%u].jmp.b_val)",
12238                         jmp_instr_id,
12239                         jmp_instr_id);
12240                 break;
12241
12242         case INSTR_JMP_GT_HI:
12243                 fprintf(f,
12244                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12245                         "pipeline_instructions[%u].jmp.b_val)",
12246                         jmp_instr_id,
12247                         jmp_instr_id);
12248                 break;
12249
12250         default:
12251                 break;
12252         }
12253
12254         /* Find the instruction group of the jump instruction. */
12255         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12256         if (!jmp_g)
12257                 return -EINVAL;
12258
12259         /* Find the instruction group of the jump destination instruction. */
12260         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12261         if (!data)
12262                 return -EINVAL;
12263
12264         instr_id = data - p->instruction_data;
12265
12266         g = instruction_group_list_group_find(igl, instr_id);
12267         if (!g)
12268                 return -EINVAL;
12269
12270         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12271          * instruction group).
12272          */
12273         if (g->group_id == jmp_g->group_id)
12274                 fprintf(f,
12275                         "\n\t\tgoto %s;\n",
12276                         jmp_data->jmp_label);
12277         else
12278                 fprintf(f,
12279                         " {\n"
12280                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12281                         "\t\treturn;\n"
12282                         "\t}\n\n",
12283                         g->group_id);
12284
12285         return 0;
12286 }
12287
12288 static void
12289 instruction_group_list_codegen(struct instruction_group_list *igl,
12290                                struct rte_swx_pipeline *p,
12291                                FILE *f)
12292 {
12293         struct instruction_group *g;
12294         uint32_t i;
12295         int is_required = 0;
12296
12297         /* Check if code generation is required. */
12298         TAILQ_FOREACH(g, igl, node)
12299                 if (g->first_instr_id < g->last_instr_id)
12300                         is_required = 1;
12301
12302         if (!is_required)
12303                 return;
12304
12305         /* Generate the code for the pipeline instruction array. */
12306         fprintf(f,
12307                 "static const struct instruction pipeline_instructions[] = {\n");
12308
12309         for (i = 0; i < p->n_instructions; i++) {
12310                 struct instruction *instr = &p->instructions[i];
12311                 instruction_export_t func = export_table[instr->type];
12312
12313                 func(instr, f);
12314         }
12315
12316         fprintf(f, "};\n\n");
12317
12318         /* Generate the code for the pipeline functions: one function for each instruction group
12319          * that contains more than one instruction.
12320          */
12321         TAILQ_FOREACH(g, igl, node) {
12322                 struct instruction *last_instr;
12323                 uint32_t j;
12324
12325                 /* Skip if group contains a single instruction. */
12326                 if (g->last_instr_id == g->first_instr_id)
12327                         continue;
12328
12329                 /* Generate new pipeline function. */
12330                 fprintf(f,
12331                         "void\n"
12332                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12333                         "{\n"
12334                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12335                         "\n",
12336                         g->group_id);
12337
12338                 /* Generate the code for each pipeline instruction. */
12339                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12340                         struct instruction *instr = &p->instructions[j];
12341                         struct instruction_data *data = &p->instruction_data[j];
12342
12343                         /* Label, if present. */
12344                         if (data->label[0])
12345                                 fprintf(f, "\n%s : ", data->label);
12346                         else
12347                                 fprintf(f, "\n\t");
12348
12349                         /* TX instruction type. */
12350                         if (instruction_does_tx(instr)) {
12351                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12352                                 continue;
12353                         }
12354
12355                         /* Jump instruction type. */
12356                         if (instruction_is_jmp(instr)) {
12357                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12358                                 continue;
12359                         }
12360
12361                         /* Any other instruction type. */
12362                         fprintf(f,
12363                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12364                                 instr_type_to_func(instr),
12365                                 j);
12366                 }
12367
12368                 /* Finalize the generated pipeline function. For some instructions such as TX,
12369                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12370                  * decided unconditionally and the instruction pointer of the current thread set
12371                  * accordingly; for all the other instructions, the instruction pointer must be
12372                  * incremented now.
12373                  */
12374                 last_instr = &p->instructions[g->last_instr_id];
12375
12376                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12377                         fprintf(f,
12378                                 "thread_ip_inc(p);\n");
12379
12380                 fprintf(f,
12381                         "}\n"
12382                         "\n");
12383         }
12384 }
12385
12386 static uint32_t
12387 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12388 {
12389         struct instruction_group *g;
12390         uint32_t n_custom_instr = 0;
12391
12392         /* Groups with a single instruction: no function is generated for this group, the group
12393          * keeps its current instruction. Groups with more than two instructions: one function and
12394          * the associated custom instruction get generated for each such group.
12395          */
12396         TAILQ_FOREACH(g, igl, node) {
12397                 if (g->first_instr_id == g->last_instr_id)
12398                         continue;
12399
12400                 n_custom_instr++;
12401         }
12402
12403         return n_custom_instr;
12404 }
12405
12406 static int
12407 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12408 {
12409         struct action *a;
12410         FILE *f = NULL;
12411
12412         /* Create the .c file. */
12413         f = fopen("/tmp/pipeline.c", "w");
12414         if (!f)
12415                 return -EIO;
12416
12417         /* Include the .h file. */
12418         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12419
12420         /* Add the code for each action. */
12421         TAILQ_FOREACH(a, &p->actions, node) {
12422                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12423
12424                 action_data_codegen(a, f);
12425
12426                 fprintf(f, "\n");
12427
12428                 action_instr_codegen(a, f);
12429
12430                 fprintf(f, "\n");
12431         }
12432
12433         /* Add the pipeline code. */
12434         instruction_group_list_codegen(igl, p, f);
12435
12436         /* Close the .c file. */
12437         fclose(f);
12438
12439         return 0;
12440 }
12441
12442 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12443 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12444 #endif
12445
12446 static int
12447 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12448 {
12449         struct action *a;
12450         struct instruction_group *g;
12451         char *dir_in, *buffer = NULL;
12452         const char *dir_out;
12453         int status = 0;
12454
12455         /* Get the environment variables. */
12456         dir_in = getenv("RTE_INSTALL_DIR");
12457         if (!dir_in) {
12458                 status = -EINVAL;
12459                 goto free;
12460         }
12461
12462         dir_out = "/tmp";
12463
12464         /* Memory allocation for the command buffer. */
12465         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12466         if (!buffer) {
12467                 status = -ENOMEM;
12468                 goto free;
12469         }
12470
12471         snprintf(buffer,
12472                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12473                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12474                  "-I %s/lib/pipeline "
12475                  "-I %s/lib/eal/include "
12476                  "-I %s/lib/eal/x86/include "
12477                  "-I %s/lib/eal/include/generic "
12478                  "-I %s/lib/meter "
12479                  "-I %s/lib/port "
12480                  "-I %s/lib/table "
12481                  "-I %s/lib/pipeline "
12482                  "-I %s/config "
12483                  "-I %s/build "
12484                  "-I %s/lib/eal/linux/include "
12485                  ">%s/pipeline.log 2>&1 "
12486                  "&& "
12487                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12488                  ">>%s/pipeline.log 2>&1",
12489                  dir_out,
12490                  dir_out,
12491                  dir_in,
12492                  dir_in,
12493                  dir_in,
12494                  dir_in,
12495                  dir_in,
12496                  dir_in,
12497                  dir_in,
12498                  dir_in,
12499                  dir_in,
12500                  dir_in,
12501                  dir_in,
12502                  dir_out,
12503                  dir_out,
12504                  dir_out,
12505                  dir_out);
12506
12507         /* Build the shared object library. */
12508         status = system(buffer);
12509         if (status)
12510                 goto free;
12511
12512         /* Open library. */
12513         snprintf(buffer,
12514                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12515                  "%s/libpipeline.so",
12516                  dir_out);
12517
12518         p->lib = dlopen(buffer, RTLD_LAZY);
12519         if (!p->lib) {
12520                 status = -EIO;
12521                 goto free;
12522         }
12523
12524         /* Get the action function symbols. */
12525         TAILQ_FOREACH(a, &p->actions, node) {
12526                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12527
12528                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12529                 if (!p->action_funcs[a->id]) {
12530                         status = -EINVAL;
12531                         goto free;
12532                 }
12533         }
12534
12535         /* Get the pipeline function symbols. */
12536         TAILQ_FOREACH(g, igl, node) {
12537                 if (g->first_instr_id == g->last_instr_id)
12538                         continue;
12539
12540                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12541
12542                 g->func = dlsym(p->lib, buffer);
12543                 if (!g->func) {
12544                         status = -EINVAL;
12545                         goto free;
12546                 }
12547         }
12548
12549 free:
12550         if (status && p->lib) {
12551                 dlclose(p->lib);
12552                 p->lib = NULL;
12553         }
12554
12555         free(buffer);
12556
12557         return status;
12558 }
12559
12560 static int
12561 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12562                       struct instruction_group_list *igl)
12563 {
12564         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12565
12566         /* Check that enough space is available within the pipeline instruction table to store all
12567          * the custom instructions.
12568          */
12569         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12570                 return -ENOSPC;
12571
12572         return 0;
12573 }
12574
12575 static void
12576 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12577 {
12578         struct instruction_group *g;
12579         uint32_t i;
12580
12581         /* Pipeline table instructions. */
12582         for (i = 0; i < p->n_instructions; i++) {
12583                 struct instruction *instr = &p->instructions[i];
12584
12585                 if (instr->type == INSTR_TABLE)
12586                         instr->type = INSTR_TABLE_AF;
12587
12588                 if (instr->type == INSTR_LEARNER)
12589                         instr->type = INSTR_LEARNER_AF;
12590         }
12591
12592         /* Pipeline custom instructions. */
12593         i = 0;
12594         TAILQ_FOREACH(g, igl, node) {
12595                 struct instruction *instr = &p->instructions[g->first_instr_id];
12596                 uint32_t j;
12597
12598                 if (g->first_instr_id == g->last_instr_id)
12599                         continue;
12600
12601                 /* Install a new custom instruction. */
12602                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12603
12604                 /* First instruction of the group: change its type to the new custom instruction. */
12605                 instr->type = INSTR_CUSTOM_0 + i;
12606
12607                 /* All the subsequent instructions of the group: invalidate. */
12608                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12609                         struct instruction_data *data = &p->instruction_data[j];
12610
12611                         data->invalid = 1;
12612                 }
12613
12614                 i++;
12615         }
12616
12617         /* Remove the invalidated instructions. */
12618         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12619
12620         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12621          * instructions that are the only instruction within their group, so they were left
12622          * unmodified).
12623          */
12624         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12625 }
12626
12627 static int
12628 pipeline_compile(struct rte_swx_pipeline *p)
12629 {
12630         struct instruction_group_list *igl = NULL;
12631         int status = 0;
12632
12633         igl = instruction_group_list_create(p);
12634         if (!igl) {
12635                 status = -ENOMEM;
12636                 goto free;
12637         }
12638
12639         /* Code generation. */
12640         status = pipeline_codegen(p, igl);
12641         if (status)
12642                 goto free;
12643
12644         /* Build and load the shared object library. */
12645         status = pipeline_libload(p, igl);
12646         if (status)
12647                 goto free;
12648
12649         /* Adjust instructions. */
12650         status = pipeline_adjust_check(p, igl);
12651         if (status)
12652                 goto free;
12653
12654         pipeline_adjust(p, igl);
12655
12656 free:
12657         instruction_group_list_free(igl);
12658
12659         return status;
12660 }