lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8 #include <dlfcn.h>
   9
  10 #include "rte_swx_pipeline_internal.h"
  11
  12 #define CHECK(condition, err_code)                                             \
  13 do {                                                                           \
  14         if (!(condition))                                                      \
  15                 return -(err_code);                                            \
  16 } while (0)
  17
  18 #define CHECK_NAME(name, err_code)                                             \
  19         CHECK((name) &&                                                        \
  20               (name)[0] &&                                                     \
  21               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  22               err_code)
  23
  24 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  25         CHECK((instr) &&                                                       \
  26               (instr)[0] &&                                                    \
  27               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  28                RTE_SWX_INSTRUCTION_SIZE),                                      \
  29               err_code)
  30
  31 /*
  32  * Environment.
  33  */
  34 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  35
  36 #include <rte_malloc.h>
  37
  38 static void *
  39 env_malloc(size_t size, size_t alignment, int numa_node)
  40 {
  41         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  42 }
  43
  44 static void
  45 env_free(void *start, size_t size __rte_unused)
  46 {
  47         rte_free(start);
  48 }
  49
  50 #else
  51
  52 #include <numa.h>
  53
  54 static void *
  55 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  56 {
  57         void *start;
  58
  59         if (numa_available() == -1)
  60                 return NULL;
  61
  62         start = numa_alloc_onnode(size, numa_node);
  63         if (!start)
  64                 return NULL;
  65
  66         memset(start, 0, size);
  67         return start;
  68 }
  69
  70 static void
  71 env_free(void *start, size_t size)
  72 {
  73         if (numa_available() == -1)
  74                 return;
  75
  76         numa_free(start, size);
  77 }
  78
  79 #endif
  80
  81 /*
  82  * Struct.
  83  */
  84 static struct struct_type *
  85 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  86 {
  87         struct struct_type *elem;
  88
  89         TAILQ_FOREACH(elem, &p->struct_types, node)
  90                 if (strcmp(elem->name, name) == 0)
  91                         return elem;
  92
  93         return NULL;
  94 }
  95
  96 static struct field *
  97 struct_type_field_find(struct struct_type *st, const char *name)
  98 {
  99         uint32_t i;
 100
 101         for (i = 0; i < st->n_fields; i++) {
 102                 struct field *f = &st->fields[i];
 103
 104                 if (strcmp(f->name, name) == 0)
 105                         return f;
 106         }
 107
 108         return NULL;
 109 }
 110
 111 int
 112 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 113                                       const char *name,
 114                                       struct rte_swx_field_params *fields,
 115                                       uint32_t n_fields,
 116                                       int last_field_has_variable_size)
 117 {
 118         struct struct_type *st;
 119         uint32_t i;
 120
 121         CHECK(p, EINVAL);
 122         CHECK_NAME(name, EINVAL);
 123         CHECK(fields, EINVAL);
 124         CHECK(n_fields, EINVAL);
 125
 126         for (i = 0; i < n_fields; i++) {
 127                 struct rte_swx_field_params *f = &fields[i];
 128                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 129                 uint32_t j;
 130
 131                 CHECK_NAME(f->name, EINVAL);
 132                 CHECK(f->n_bits, EINVAL);
 133                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 134                 CHECK((f->n_bits & 7) == 0, EINVAL);
 135
 136                 for (j = 0; j < i; j++) {
 137                         struct rte_swx_field_params *f_prev = &fields[j];
 138
 139                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 140                 }
 141         }
 142
 143         CHECK(!struct_type_find(p, name), EEXIST);
 144
 145         /* Node allocation. */
 146         st = calloc(1, sizeof(struct struct_type));
 147         CHECK(st, ENOMEM);
 148
 149         st->fields = calloc(n_fields, sizeof(struct field));
 150         if (!st->fields) {
 151                 free(st);
 152                 CHECK(0, ENOMEM);
 153         }
 154
 155         /* Node initialization. */
 156         strcpy(st->name, name);
 157         for (i = 0; i < n_fields; i++) {
 158                 struct field *dst = &st->fields[i];
 159                 struct rte_swx_field_params *src = &fields[i];
 160                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 161
 162                 strcpy(dst->name, src->name);
 163                 dst->n_bits = src->n_bits;
 164                 dst->offset = st->n_bits;
 165                 dst->var_size = var_size;
 166
 167                 st->n_bits += src->n_bits;
 168                 st->n_bits_min += var_size ? 0 : src->n_bits;
 169         }
 170         st->n_fields = n_fields;
 171         st->var_size = last_field_has_variable_size;
 172
 173         /* Node add to tailq. */
 174         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 175
 176         return 0;
 177 }
 178
 179 static int
 180 struct_build(struct rte_swx_pipeline *p)
 181 {
 182         uint32_t i;
 183
 184         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 185                 struct thread *t = &p->threads[i];
 186
 187                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 188                 CHECK(t->structs, ENOMEM);
 189         }
 190
 191         return 0;
 192 }
 193
 194 static void
 195 struct_build_free(struct rte_swx_pipeline *p)
 196 {
 197         uint32_t i;
 198
 199         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 200                 struct thread *t = &p->threads[i];
 201
 202                 free(t->structs);
 203                 t->structs = NULL;
 204         }
 205 }
 206
 207 static void
 208 struct_free(struct rte_swx_pipeline *p)
 209 {
 210         struct_build_free(p);
 211
 212         /* Struct types. */
 213         for ( ; ; ) {
 214                 struct struct_type *elem;
 215
 216                 elem = TAILQ_FIRST(&p->struct_types);
 217                 if (!elem)
 218                         break;
 219
 220                 TAILQ_REMOVE(&p->struct_types, elem, node);
 221                 free(elem->fields);
 222                 free(elem);
 223         }
 224 }
 225
 226 /*
 227  * Input port.
 228  */
 229 static struct port_in_type *
 230 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 231 {
 232         struct port_in_type *elem;
 233
 234         if (!name)
 235                 return NULL;
 236
 237         TAILQ_FOREACH(elem, &p->port_in_types, node)
 238                 if (strcmp(elem->name, name) == 0)
 239                         return elem;
 240
 241         return NULL;
 242 }
 243
 244 int
 245 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 246                                        const char *name,
 247                                        struct rte_swx_port_in_ops *ops)
 248 {
 249         struct port_in_type *elem;
 250
 251         CHECK(p, EINVAL);
 252         CHECK_NAME(name, EINVAL);
 253         CHECK(ops, EINVAL);
 254         CHECK(ops->create, EINVAL);
 255         CHECK(ops->free, EINVAL);
 256         CHECK(ops->pkt_rx, EINVAL);
 257         CHECK(ops->stats_read, EINVAL);
 258
 259         CHECK(!port_in_type_find(p, name), EEXIST);
 260
 261         /* Node allocation. */
 262         elem = calloc(1, sizeof(struct port_in_type));
 263         CHECK(elem, ENOMEM);
 264
 265         /* Node initialization. */
 266         strcpy(elem->name, name);
 267         memcpy(&elem->ops, ops, sizeof(*ops));
 268
 269         /* Node add to tailq. */
 270         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 271
 272         return 0;
 273 }
 274
 275 static struct port_in *
 276 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 277 {
 278         struct port_in *port;
 279
 280         TAILQ_FOREACH(port, &p->ports_in, node)
 281                 if (port->id == port_id)
 282                         return port;
 283
 284         return NULL;
 285 }
 286
 287 int
 288 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 289                                 uint32_t port_id,
 290                                 const char *port_type_name,
 291                                 void *args)
 292 {
 293         struct port_in_type *type = NULL;
 294         struct port_in *port = NULL;
 295         void *obj = NULL;
 296
 297         CHECK(p, EINVAL);
 298
 299         CHECK(!port_in_find(p, port_id), EINVAL);
 300
 301         CHECK_NAME(port_type_name, EINVAL);
 302         type = port_in_type_find(p, port_type_name);
 303         CHECK(type, EINVAL);
 304
 305         obj = type->ops.create(args);
 306         CHECK(obj, ENODEV);
 307
 308         /* Node allocation. */
 309         port = calloc(1, sizeof(struct port_in));
 310         CHECK(port, ENOMEM);
 311
 312         /* Node initialization. */
 313         port->type = type;
 314         port->obj = obj;
 315         port->id = port_id;
 316
 317         /* Node add to tailq. */
 318         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 319         if (p->n_ports_in < port_id + 1)
 320                 p->n_ports_in = port_id + 1;
 321
 322         return 0;
 323 }
 324
 325 static int
 326 port_in_build(struct rte_swx_pipeline *p)
 327 {
 328         struct port_in *port;
 329         uint32_t i;
 330
 331         CHECK(p->n_ports_in, EINVAL);
 332         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 333
 334         for (i = 0; i < p->n_ports_in; i++)
 335                 CHECK(port_in_find(p, i), EINVAL);
 336
 337         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 338         CHECK(p->in, ENOMEM);
 339
 340         TAILQ_FOREACH(port, &p->ports_in, node) {
 341                 struct port_in_runtime *in = &p->in[port->id];
 342
 343                 in->pkt_rx = port->type->ops.pkt_rx;
 344                 in->obj = port->obj;
 345         }
 346
 347         return 0;
 348 }
 349
 350 static void
 351 port_in_build_free(struct rte_swx_pipeline *p)
 352 {
 353         free(p->in);
 354         p->in = NULL;
 355 }
 356
 357 static void
 358 port_in_free(struct rte_swx_pipeline *p)
 359 {
 360         port_in_build_free(p);
 361
 362         /* Input ports. */
 363         for ( ; ; ) {
 364                 struct port_in *port;
 365
 366                 port = TAILQ_FIRST(&p->ports_in);
 367                 if (!port)
 368                         break;
 369
 370                 TAILQ_REMOVE(&p->ports_in, port, node);
 371                 port->type->ops.free(port->obj);
 372                 free(port);
 373         }
 374
 375         /* Input port types. */
 376         for ( ; ; ) {
 377                 struct port_in_type *elem;
 378
 379                 elem = TAILQ_FIRST(&p->port_in_types);
 380                 if (!elem)
 381                         break;
 382
 383                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 384                 free(elem);
 385         }
 386 }
 387
 388 /*
 389  * Output port.
 390  */
 391 static struct port_out_type *
 392 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 393 {
 394         struct port_out_type *elem;
 395
 396         if (!name)
 397                 return NULL;
 398
 399         TAILQ_FOREACH(elem, &p->port_out_types, node)
 400                 if (!strcmp(elem->name, name))
 401                         return elem;
 402
 403         return NULL;
 404 }
 405
 406 int
 407 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 408                                         const char *name,
 409                                         struct rte_swx_port_out_ops *ops)
 410 {
 411         struct port_out_type *elem;
 412
 413         CHECK(p, EINVAL);
 414         CHECK_NAME(name, EINVAL);
 415         CHECK(ops, EINVAL);
 416         CHECK(ops->create, EINVAL);
 417         CHECK(ops->free, EINVAL);
 418         CHECK(ops->pkt_tx, EINVAL);
 419         CHECK(ops->stats_read, EINVAL);
 420
 421         CHECK(!port_out_type_find(p, name), EEXIST);
 422
 423         /* Node allocation. */
 424         elem = calloc(1, sizeof(struct port_out_type));
 425         CHECK(elem, ENOMEM);
 426
 427         /* Node initialization. */
 428         strcpy(elem->name, name);
 429         memcpy(&elem->ops, ops, sizeof(*ops));
 430
 431         /* Node add to tailq. */
 432         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 433
 434         return 0;
 435 }
 436
 437 static struct port_out *
 438 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 439 {
 440         struct port_out *port;
 441
 442         TAILQ_FOREACH(port, &p->ports_out, node)
 443                 if (port->id == port_id)
 444                         return port;
 445
 446         return NULL;
 447 }
 448
 449 int
 450 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 451                                  uint32_t port_id,
 452                                  const char *port_type_name,
 453                                  void *args)
 454 {
 455         struct port_out_type *type = NULL;
 456         struct port_out *port = NULL;
 457         void *obj = NULL;
 458
 459         CHECK(p, EINVAL);
 460
 461         CHECK(!port_out_find(p, port_id), EINVAL);
 462
 463         CHECK_NAME(port_type_name, EINVAL);
 464         type = port_out_type_find(p, port_type_name);
 465         CHECK(type, EINVAL);
 466
 467         obj = type->ops.create(args);
 468         CHECK(obj, ENODEV);
 469
 470         /* Node allocation. */
 471         port = calloc(1, sizeof(struct port_out));
 472         CHECK(port, ENOMEM);
 473
 474         /* Node initialization. */
 475         port->type = type;
 476         port->obj = obj;
 477         port->id = port_id;
 478
 479         /* Node add to tailq. */
 480         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 481         if (p->n_ports_out < port_id + 1)
 482                 p->n_ports_out = port_id + 1;
 483
 484         return 0;
 485 }
 486
 487 static int
 488 port_out_build(struct rte_swx_pipeline *p)
 489 {
 490         struct port_out *port;
 491         uint32_t i;
 492
 493         CHECK(p->n_ports_out, EINVAL);
 494
 495         for (i = 0; i < p->n_ports_out; i++)
 496                 CHECK(port_out_find(p, i), EINVAL);
 497
 498         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 499         CHECK(p->out, ENOMEM);
 500
 501         TAILQ_FOREACH(port, &p->ports_out, node) {
 502                 struct port_out_runtime *out = &p->out[port->id];
 503
 504                 out->pkt_tx = port->type->ops.pkt_tx;
 505                 out->flush = port->type->ops.flush;
 506                 out->obj = port->obj;
 507         }
 508
 509         return 0;
 510 }
 511
 512 static void
 513 port_out_build_free(struct rte_swx_pipeline *p)
 514 {
 515         free(p->out);
 516         p->out = NULL;
 517 }
 518
 519 static void
 520 port_out_free(struct rte_swx_pipeline *p)
 521 {
 522         port_out_build_free(p);
 523
 524         /* Output ports. */
 525         for ( ; ; ) {
 526                 struct port_out *port;
 527
 528                 port = TAILQ_FIRST(&p->ports_out);
 529                 if (!port)
 530                         break;
 531
 532                 TAILQ_REMOVE(&p->ports_out, port, node);
 533                 port->type->ops.free(port->obj);
 534                 free(port);
 535         }
 536
 537         /* Output port types. */
 538         for ( ; ; ) {
 539                 struct port_out_type *elem;
 540
 541                 elem = TAILQ_FIRST(&p->port_out_types);
 542                 if (!elem)
 543                         break;
 544
 545                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 546                 free(elem);
 547         }
 548 }
 549
 550 /*
 551  * Extern object.
 552  */
 553 static struct extern_type *
 554 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 555 {
 556         struct extern_type *elem;
 557
 558         TAILQ_FOREACH(elem, &p->extern_types, node)
 559                 if (strcmp(elem->name, name) == 0)
 560                         return elem;
 561
 562         return NULL;
 563 }
 564
 565 static struct extern_type_member_func *
 566 extern_type_member_func_find(struct extern_type *type, const char *name)
 567 {
 568         struct extern_type_member_func *elem;
 569
 570         TAILQ_FOREACH(elem, &type->funcs, node)
 571                 if (strcmp(elem->name, name) == 0)
 572                         return elem;
 573
 574         return NULL;
 575 }
 576
 577 static struct extern_obj *
 578 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 579 {
 580         struct extern_obj *elem;
 581
 582         TAILQ_FOREACH(elem, &p->extern_objs, node)
 583                 if (strcmp(elem->name, name) == 0)
 584                         return elem;
 585
 586         return NULL;
 587 }
 588
 589 static struct extern_type_member_func *
 590 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 591                              const char *name,
 592                              struct extern_obj **obj)
 593 {
 594         struct extern_obj *object;
 595         struct extern_type_member_func *func;
 596         char *object_name, *func_name;
 597
 598         if (name[0] != 'e' || name[1] != '.')
 599                 return NULL;
 600
 601         object_name = strdup(&name[2]);
 602         if (!object_name)
 603                 return NULL;
 604
 605         func_name = strchr(object_name, '.');
 606         if (!func_name) {
 607                 free(object_name);
 608                 return NULL;
 609         }
 610
 611         *func_name = 0;
 612         func_name++;
 613
 614         object = extern_obj_find(p, object_name);
 615         if (!object) {
 616                 free(object_name);
 617                 return NULL;
 618         }
 619
 620         func = extern_type_member_func_find(object->type, func_name);
 621         if (!func) {
 622                 free(object_name);
 623                 return NULL;
 624         }
 625
 626         if (obj)
 627                 *obj = object;
 628
 629         free(object_name);
 630         return func;
 631 }
 632
 633 static struct field *
 634 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 635                                const char *name,
 636                                struct extern_obj **object)
 637 {
 638         struct extern_obj *obj;
 639         struct field *f;
 640         char *obj_name, *field_name;
 641
 642         if ((name[0] != 'e') || (name[1] != '.'))
 643                 return NULL;
 644
 645         obj_name = strdup(&name[2]);
 646         if (!obj_name)
 647                 return NULL;
 648
 649         field_name = strchr(obj_name, '.');
 650         if (!field_name) {
 651                 free(obj_name);
 652                 return NULL;
 653         }
 654
 655         *field_name = 0;
 656         field_name++;
 657
 658         obj = extern_obj_find(p, obj_name);
 659         if (!obj) {
 660                 free(obj_name);
 661                 return NULL;
 662         }
 663
 664         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 665         if (!f) {
 666                 free(obj_name);
 667                 return NULL;
 668         }
 669
 670         if (object)
 671                 *object = obj;
 672
 673         free(obj_name);
 674         return f;
 675 }
 676
 677 int
 678 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 679         const char *name,
 680         const char *mailbox_struct_type_name,
 681         rte_swx_extern_type_constructor_t constructor,
 682         rte_swx_extern_type_destructor_t destructor)
 683 {
 684         struct extern_type *elem;
 685         struct struct_type *mailbox_struct_type;
 686
 687         CHECK(p, EINVAL);
 688
 689         CHECK_NAME(name, EINVAL);
 690         CHECK(!extern_type_find(p, name), EEXIST);
 691
 692         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 693         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 694         CHECK(mailbox_struct_type, EINVAL);
 695         CHECK(!mailbox_struct_type->var_size, EINVAL);
 696
 697         CHECK(constructor, EINVAL);
 698         CHECK(destructor, EINVAL);
 699
 700         /* Node allocation. */
 701         elem = calloc(1, sizeof(struct extern_type));
 702         CHECK(elem, ENOMEM);
 703
 704         /* Node initialization. */
 705         strcpy(elem->name, name);
 706         elem->mailbox_struct_type = mailbox_struct_type;
 707         elem->constructor = constructor;
 708         elem->destructor = destructor;
 709         TAILQ_INIT(&elem->funcs);
 710
 711         /* Node add to tailq. */
 712         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 713
 714         return 0;
 715 }
 716
 717 int
 718 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 719         const char *extern_type_name,
 720         const char *name,
 721         rte_swx_extern_type_member_func_t member_func)
 722 {
 723         struct extern_type *type;
 724         struct extern_type_member_func *type_member;
 725
 726         CHECK(p, EINVAL);
 727
 728         CHECK_NAME(extern_type_name, EINVAL);
 729         type = extern_type_find(p, extern_type_name);
 730         CHECK(type, EINVAL);
 731         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 732
 733         CHECK_NAME(name, EINVAL);
 734         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 735
 736         CHECK(member_func, EINVAL);
 737
 738         /* Node allocation. */
 739         type_member = calloc(1, sizeof(struct extern_type_member_func));
 740         CHECK(type_member, ENOMEM);
 741
 742         /* Node initialization. */
 743         strcpy(type_member->name, name);
 744         type_member->func = member_func;
 745         type_member->id = type->n_funcs;
 746
 747         /* Node add to tailq. */
 748         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 749         type->n_funcs++;
 750
 751         return 0;
 752 }
 753
 754 int
 755 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 756                                       const char *extern_type_name,
 757                                       const char *name,
 758                                       const char *args)
 759 {
 760         struct extern_type *type;
 761         struct extern_obj *obj;
 762         void *obj_handle;
 763
 764         CHECK(p, EINVAL);
 765
 766         CHECK_NAME(extern_type_name, EINVAL);
 767         type = extern_type_find(p, extern_type_name);
 768         CHECK(type, EINVAL);
 769
 770         CHECK_NAME(name, EINVAL);
 771         CHECK(!extern_obj_find(p, name), EEXIST);
 772
 773         /* Node allocation. */
 774         obj = calloc(1, sizeof(struct extern_obj));
 775         CHECK(obj, ENOMEM);
 776
 777         /* Object construction. */
 778         obj_handle = type->constructor(args);
 779         if (!obj_handle) {
 780                 free(obj);
 781                 CHECK(0, ENODEV);
 782         }
 783
 784         /* Node initialization. */
 785         strcpy(obj->name, name);
 786         obj->type = type;
 787         obj->obj = obj_handle;
 788         obj->struct_id = p->n_structs;
 789         obj->id = p->n_extern_objs;
 790
 791         /* Node add to tailq. */
 792         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 793         p->n_extern_objs++;
 794         p->n_structs++;
 795
 796         return 0;
 797 }
 798
 799 static int
 800 extern_obj_build(struct rte_swx_pipeline *p)
 801 {
 802         uint32_t i;
 803
 804         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 805                 struct thread *t = &p->threads[i];
 806                 struct extern_obj *obj;
 807
 808                 t->extern_objs = calloc(p->n_extern_objs,
 809                                         sizeof(struct extern_obj_runtime));
 810                 CHECK(t->extern_objs, ENOMEM);
 811
 812                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 813                         struct extern_obj_runtime *r =
 814                                 &t->extern_objs[obj->id];
 815                         struct extern_type_member_func *func;
 816                         uint32_t mailbox_size =
 817                                 obj->type->mailbox_struct_type->n_bits / 8;
 818
 819                         r->obj = obj->obj;
 820
 821                         r->mailbox = calloc(1, mailbox_size);
 822                         CHECK(r->mailbox, ENOMEM);
 823
 824                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 825                                 r->funcs[func->id] = func->func;
 826
 827                         t->structs[obj->struct_id] = r->mailbox;
 828                 }
 829         }
 830
 831         return 0;
 832 }
 833
 834 static void
 835 extern_obj_build_free(struct rte_swx_pipeline *p)
 836 {
 837         uint32_t i;
 838
 839         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 840                 struct thread *t = &p->threads[i];
 841                 uint32_t j;
 842
 843                 if (!t->extern_objs)
 844                         continue;
 845
 846                 for (j = 0; j < p->n_extern_objs; j++) {
 847                         struct extern_obj_runtime *r = &t->extern_objs[j];
 848
 849                         free(r->mailbox);
 850                 }
 851
 852                 free(t->extern_objs);
 853                 t->extern_objs = NULL;
 854         }
 855 }
 856
 857 static void
 858 extern_obj_free(struct rte_swx_pipeline *p)
 859 {
 860         extern_obj_build_free(p);
 861
 862         /* Extern objects. */
 863         for ( ; ; ) {
 864                 struct extern_obj *elem;
 865
 866                 elem = TAILQ_FIRST(&p->extern_objs);
 867                 if (!elem)
 868                         break;
 869
 870                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 871                 if (elem->obj)
 872                         elem->type->destructor(elem->obj);
 873                 free(elem);
 874         }
 875
 876         /* Extern types. */
 877         for ( ; ; ) {
 878                 struct extern_type *elem;
 879
 880                 elem = TAILQ_FIRST(&p->extern_types);
 881                 if (!elem)
 882                         break;
 883
 884                 TAILQ_REMOVE(&p->extern_types, elem, node);
 885
 886                 for ( ; ; ) {
 887                         struct extern_type_member_func *func;
 888
 889                         func = TAILQ_FIRST(&elem->funcs);
 890                         if (!func)
 891                                 break;
 892
 893                         TAILQ_REMOVE(&elem->funcs, func, node);
 894                         free(func);
 895                 }
 896
 897                 free(elem);
 898         }
 899 }
 900
 901 /*
 902  * Extern function.
 903  */
 904 static struct extern_func *
 905 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 906 {
 907         struct extern_func *elem;
 908
 909         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 910                 if (strcmp(elem->name, name) == 0)
 911                         return elem;
 912
 913         return NULL;
 914 }
 915
 916 static struct extern_func *
 917 extern_func_parse(struct rte_swx_pipeline *p,
 918                   const char *name)
 919 {
 920         if (name[0] != 'f' || name[1] != '.')
 921                 return NULL;
 922
 923         return extern_func_find(p, &name[2]);
 924 }
 925
 926 static struct field *
 927 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 928                                 const char *name,
 929                                 struct extern_func **function)
 930 {
 931         struct extern_func *func;
 932         struct field *f;
 933         char *func_name, *field_name;
 934
 935         if ((name[0] != 'f') || (name[1] != '.'))
 936                 return NULL;
 937
 938         func_name = strdup(&name[2]);
 939         if (!func_name)
 940                 return NULL;
 941
 942         field_name = strchr(func_name, '.');
 943         if (!field_name) {
 944                 free(func_name);
 945                 return NULL;
 946         }
 947
 948         *field_name = 0;
 949         field_name++;
 950
 951         func = extern_func_find(p, func_name);
 952         if (!func) {
 953                 free(func_name);
 954                 return NULL;
 955         }
 956
 957         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 958         if (!f) {
 959                 free(func_name);
 960                 return NULL;
 961         }
 962
 963         if (function)
 964                 *function = func;
 965
 966         free(func_name);
 967         return f;
 968 }
 969
 970 int
 971 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 972                                       const char *name,
 973                                       const char *mailbox_struct_type_name,
 974                                       rte_swx_extern_func_t func)
 975 {
 976         struct extern_func *f;
 977         struct struct_type *mailbox_struct_type;
 978
 979         CHECK(p, EINVAL);
 980
 981         CHECK_NAME(name, EINVAL);
 982         CHECK(!extern_func_find(p, name), EEXIST);
 983
 984         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 985         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 986         CHECK(mailbox_struct_type, EINVAL);
 987         CHECK(!mailbox_struct_type->var_size, EINVAL);
 988
 989         CHECK(func, EINVAL);
 990
 991         /* Node allocation. */
 992         f = calloc(1, sizeof(struct extern_func));
 993         CHECK(func, ENOMEM);
 994
 995         /* Node initialization. */
 996         strcpy(f->name, name);
 997         f->mailbox_struct_type = mailbox_struct_type;
 998         f->func = func;
 999         f->struct_id = p->n_structs;
1000         f->id = p->n_extern_funcs;
1001
1002         /* Node add to tailq. */
1003         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1004         p->n_extern_funcs++;
1005         p->n_structs++;
1006
1007         return 0;
1008 }
1009
1010 static int
1011 extern_func_build(struct rte_swx_pipeline *p)
1012 {
1013         uint32_t i;
1014
1015         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1016                 struct thread *t = &p->threads[i];
1017                 struct extern_func *func;
1018
1019                 /* Memory allocation. */
1020                 t->extern_funcs = calloc(p->n_extern_funcs,
1021                                          sizeof(struct extern_func_runtime));
1022                 CHECK(t->extern_funcs, ENOMEM);
1023
1024                 /* Extern function. */
1025                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1026                         struct extern_func_runtime *r =
1027                                 &t->extern_funcs[func->id];
1028                         uint32_t mailbox_size =
1029                                 func->mailbox_struct_type->n_bits / 8;
1030
1031                         r->func = func->func;
1032
1033                         r->mailbox = calloc(1, mailbox_size);
1034                         CHECK(r->mailbox, ENOMEM);
1035
1036                         t->structs[func->struct_id] = r->mailbox;
1037                 }
1038         }
1039
1040         return 0;
1041 }
1042
1043 static void
1044 extern_func_build_free(struct rte_swx_pipeline *p)
1045 {
1046         uint32_t i;
1047
1048         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1049                 struct thread *t = &p->threads[i];
1050                 uint32_t j;
1051
1052                 if (!t->extern_funcs)
1053                         continue;
1054
1055                 for (j = 0; j < p->n_extern_funcs; j++) {
1056                         struct extern_func_runtime *r = &t->extern_funcs[j];
1057
1058                         free(r->mailbox);
1059                 }
1060
1061                 free(t->extern_funcs);
1062                 t->extern_funcs = NULL;
1063         }
1064 }
1065
1066 static void
1067 extern_func_free(struct rte_swx_pipeline *p)
1068 {
1069         extern_func_build_free(p);
1070
1071         for ( ; ; ) {
1072                 struct extern_func *elem;
1073
1074                 elem = TAILQ_FIRST(&p->extern_funcs);
1075                 if (!elem)
1076                         break;
1077
1078                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1079                 free(elem);
1080         }
1081 }
1082
1083 /*
1084  * Header.
1085  */
1086 static struct header *
1087 header_find(struct rte_swx_pipeline *p, const char *name)
1088 {
1089         struct header *elem;
1090
1091         TAILQ_FOREACH(elem, &p->headers, node)
1092                 if (strcmp(elem->name, name) == 0)
1093                         return elem;
1094
1095         return NULL;
1096 }
1097
1098 static struct header *
1099 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1100 {
1101         struct header *elem;
1102
1103         TAILQ_FOREACH(elem, &p->headers, node)
1104                 if (elem->struct_id == struct_id)
1105                         return elem;
1106
1107         return NULL;
1108 }
1109
1110 static struct header *
1111 header_parse(struct rte_swx_pipeline *p,
1112              const char *name)
1113 {
1114         if (name[0] != 'h' || name[1] != '.')
1115                 return NULL;
1116
1117         return header_find(p, &name[2]);
1118 }
1119
1120 static struct field *
1121 header_field_parse(struct rte_swx_pipeline *p,
1122                    const char *name,
1123                    struct header **header)
1124 {
1125         struct header *h;
1126         struct field *f;
1127         char *header_name, *field_name;
1128
1129         if ((name[0] != 'h') || (name[1] != '.'))
1130                 return NULL;
1131
1132         header_name = strdup(&name[2]);
1133         if (!header_name)
1134                 return NULL;
1135
1136         field_name = strchr(header_name, '.');
1137         if (!field_name) {
1138                 free(header_name);
1139                 return NULL;
1140         }
1141
1142         *field_name = 0;
1143         field_name++;
1144
1145         h = header_find(p, header_name);
1146         if (!h) {
1147                 free(header_name);
1148                 return NULL;
1149         }
1150
1151         f = struct_type_field_find(h->st, field_name);
1152         if (!f) {
1153                 free(header_name);
1154                 return NULL;
1155         }
1156
1157         if (header)
1158                 *header = h;
1159
1160         free(header_name);
1161         return f;
1162 }
1163
1164 int
1165 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1166                                         const char *name,
1167                                         const char *struct_type_name)
1168 {
1169         struct struct_type *st;
1170         struct header *h;
1171         size_t n_headers_max;
1172
1173         CHECK(p, EINVAL);
1174         CHECK_NAME(name, EINVAL);
1175         CHECK_NAME(struct_type_name, EINVAL);
1176
1177         CHECK(!header_find(p, name), EEXIST);
1178
1179         st = struct_type_find(p, struct_type_name);
1180         CHECK(st, EINVAL);
1181
1182         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1183         CHECK(p->n_headers < n_headers_max, ENOSPC);
1184
1185         /* Node allocation. */
1186         h = calloc(1, sizeof(struct header));
1187         CHECK(h, ENOMEM);
1188
1189         /* Node initialization. */
1190         strcpy(h->name, name);
1191         h->st = st;
1192         h->struct_id = p->n_structs;
1193         h->id = p->n_headers;
1194
1195         /* Node add to tailq. */
1196         TAILQ_INSERT_TAIL(&p->headers, h, node);
1197         p->n_headers++;
1198         p->n_structs++;
1199
1200         return 0;
1201 }
1202
1203 static int
1204 header_build(struct rte_swx_pipeline *p)
1205 {
1206         struct header *h;
1207         uint32_t n_bytes = 0, i;
1208
1209         TAILQ_FOREACH(h, &p->headers, node) {
1210                 n_bytes += h->st->n_bits / 8;
1211         }
1212
1213         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1214                 struct thread *t = &p->threads[i];
1215                 uint32_t offset = 0;
1216
1217                 t->headers = calloc(p->n_headers,
1218                                     sizeof(struct header_runtime));
1219                 CHECK(t->headers, ENOMEM);
1220
1221                 t->headers_out = calloc(p->n_headers,
1222                                         sizeof(struct header_out_runtime));
1223                 CHECK(t->headers_out, ENOMEM);
1224
1225                 t->header_storage = calloc(1, n_bytes);
1226                 CHECK(t->header_storage, ENOMEM);
1227
1228                 t->header_out_storage = calloc(1, n_bytes);
1229                 CHECK(t->header_out_storage, ENOMEM);
1230
1231                 TAILQ_FOREACH(h, &p->headers, node) {
1232                         uint8_t *header_storage;
1233                         uint32_t n_bytes =  h->st->n_bits / 8;
1234
1235                         header_storage = &t->header_storage[offset];
1236                         offset += n_bytes;
1237
1238                         t->headers[h->id].ptr0 = header_storage;
1239                         t->headers[h->id].n_bytes = n_bytes;
1240
1241                         t->structs[h->struct_id] = header_storage;
1242                 }
1243         }
1244
1245         return 0;
1246 }
1247
1248 static void
1249 header_build_free(struct rte_swx_pipeline *p)
1250 {
1251         uint32_t i;
1252
1253         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1254                 struct thread *t = &p->threads[i];
1255
1256                 free(t->headers_out);
1257                 t->headers_out = NULL;
1258
1259                 free(t->headers);
1260                 t->headers = NULL;
1261
1262                 free(t->header_out_storage);
1263                 t->header_out_storage = NULL;
1264
1265                 free(t->header_storage);
1266                 t->header_storage = NULL;
1267         }
1268 }
1269
1270 static void
1271 header_free(struct rte_swx_pipeline *p)
1272 {
1273         header_build_free(p);
1274
1275         for ( ; ; ) {
1276                 struct header *elem;
1277
1278                 elem = TAILQ_FIRST(&p->headers);
1279                 if (!elem)
1280                         break;
1281
1282                 TAILQ_REMOVE(&p->headers, elem, node);
1283                 free(elem);
1284         }
1285 }
1286
1287 /*
1288  * Meta-data.
1289  */
1290 static struct field *
1291 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1292 {
1293         if (!p->metadata_st)
1294                 return NULL;
1295
1296         if (name[0] != 'm' || name[1] != '.')
1297                 return NULL;
1298
1299         return struct_type_field_find(p->metadata_st, &name[2]);
1300 }
1301
1302 int
1303 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1304                                           const char *struct_type_name)
1305 {
1306         struct struct_type *st = NULL;
1307
1308         CHECK(p, EINVAL);
1309
1310         CHECK_NAME(struct_type_name, EINVAL);
1311         st  = struct_type_find(p, struct_type_name);
1312         CHECK(st, EINVAL);
1313         CHECK(!st->var_size, EINVAL);
1314         CHECK(!p->metadata_st, EINVAL);
1315
1316         p->metadata_st = st;
1317         p->metadata_struct_id = p->n_structs;
1318
1319         p->n_structs++;
1320
1321         return 0;
1322 }
1323
1324 static int
1325 metadata_build(struct rte_swx_pipeline *p)
1326 {
1327         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1328         uint32_t i;
1329
1330         /* Thread-level initialization. */
1331         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1332                 struct thread *t = &p->threads[i];
1333                 uint8_t *metadata;
1334
1335                 metadata = calloc(1, n_bytes);
1336                 CHECK(metadata, ENOMEM);
1337
1338                 t->metadata = metadata;
1339                 t->structs[p->metadata_struct_id] = metadata;
1340         }
1341
1342         return 0;
1343 }
1344
1345 static void
1346 metadata_build_free(struct rte_swx_pipeline *p)
1347 {
1348         uint32_t i;
1349
1350         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1351                 struct thread *t = &p->threads[i];
1352
1353                 free(t->metadata);
1354                 t->metadata = NULL;
1355         }
1356 }
1357
1358 static void
1359 metadata_free(struct rte_swx_pipeline *p)
1360 {
1361         metadata_build_free(p);
1362 }
1363
1364 /*
1365  * Instruction.
1366  */
1367 static int
1368 instruction_is_tx(enum instruction_type type)
1369 {
1370         switch (type) {
1371         case INSTR_TX:
1372         case INSTR_TX_I:
1373                 return 1;
1374
1375         default:
1376                 return 0;
1377         }
1378 }
1379
1380 static int
1381 instruction_does_tx(struct instruction *instr)
1382 {
1383         switch (instr->type) {
1384         case INSTR_TX:
1385         case INSTR_TX_I:
1386         case INSTR_HDR_EMIT_TX:
1387         case INSTR_HDR_EMIT2_TX:
1388         case INSTR_HDR_EMIT3_TX:
1389         case INSTR_HDR_EMIT4_TX:
1390         case INSTR_HDR_EMIT5_TX:
1391         case INSTR_HDR_EMIT6_TX:
1392         case INSTR_HDR_EMIT7_TX:
1393         case INSTR_HDR_EMIT8_TX:
1394                 return 1;
1395         default:
1396                 return 0;
1397         }
1398 }
1399
1400 static int
1401 instruction_is_jmp(struct instruction *instr)
1402 {
1403         switch (instr->type) {
1404         case INSTR_JMP:
1405         case INSTR_JMP_VALID:
1406         case INSTR_JMP_INVALID:
1407         case INSTR_JMP_HIT:
1408         case INSTR_JMP_MISS:
1409         case INSTR_JMP_ACTION_HIT:
1410         case INSTR_JMP_ACTION_MISS:
1411         case INSTR_JMP_EQ:
1412         case INSTR_JMP_EQ_MH:
1413         case INSTR_JMP_EQ_HM:
1414         case INSTR_JMP_EQ_HH:
1415         case INSTR_JMP_EQ_I:
1416         case INSTR_JMP_NEQ:
1417         case INSTR_JMP_NEQ_MH:
1418         case INSTR_JMP_NEQ_HM:
1419         case INSTR_JMP_NEQ_HH:
1420         case INSTR_JMP_NEQ_I:
1421         case INSTR_JMP_LT:
1422         case INSTR_JMP_LT_MH:
1423         case INSTR_JMP_LT_HM:
1424         case INSTR_JMP_LT_HH:
1425         case INSTR_JMP_LT_MI:
1426         case INSTR_JMP_LT_HI:
1427         case INSTR_JMP_GT:
1428         case INSTR_JMP_GT_MH:
1429         case INSTR_JMP_GT_HM:
1430         case INSTR_JMP_GT_HH:
1431         case INSTR_JMP_GT_MI:
1432         case INSTR_JMP_GT_HI:
1433                 return 1;
1434
1435         default:
1436                 return 0;
1437         }
1438 }
1439
1440 static int
1441 instruction_does_thread_yield(struct instruction *instr)
1442 {
1443         switch (instr->type) {
1444         case INSTR_RX:
1445         case INSTR_TABLE:
1446         case INSTR_TABLE_AF:
1447         case INSTR_SELECTOR:
1448         case INSTR_LEARNER:
1449         case INSTR_LEARNER_AF:
1450         case INSTR_EXTERN_OBJ:
1451         case INSTR_EXTERN_FUNC:
1452                 return 1;
1453         default:
1454                 return 0;
1455         }
1456 }
1457
1458 static struct field *
1459 action_field_parse(struct action *action, const char *name);
1460
1461 static struct field *
1462 struct_field_parse(struct rte_swx_pipeline *p,
1463                    struct action *action,
1464                    const char *name,
1465                    uint32_t *struct_id)
1466 {
1467         struct field *f;
1468
1469         switch (name[0]) {
1470         case 'h':
1471         {
1472                 struct header *header;
1473
1474                 f = header_field_parse(p, name, &header);
1475                 if (!f)
1476                         return NULL;
1477
1478                 *struct_id = header->struct_id;
1479                 return f;
1480         }
1481
1482         case 'm':
1483         {
1484                 f = metadata_field_parse(p, name);
1485                 if (!f)
1486                         return NULL;
1487
1488                 *struct_id = p->metadata_struct_id;
1489                 return f;
1490         }
1491
1492         case 't':
1493         {
1494                 if (!action)
1495                         return NULL;
1496
1497                 f = action_field_parse(action, name);
1498                 if (!f)
1499                         return NULL;
1500
1501                 *struct_id = 0;
1502                 return f;
1503         }
1504
1505         case 'e':
1506         {
1507                 struct extern_obj *obj;
1508
1509                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1510                 if (!f)
1511                         return NULL;
1512
1513                 *struct_id = obj->struct_id;
1514                 return f;
1515         }
1516
1517         case 'f':
1518         {
1519                 struct extern_func *func;
1520
1521                 f = extern_func_mailbox_field_parse(p, name, &func);
1522                 if (!f)
1523                         return NULL;
1524
1525                 *struct_id = func->struct_id;
1526                 return f;
1527         }
1528
1529         default:
1530                 return NULL;
1531         }
1532 }
1533
1534 /*
1535  * rx.
1536  */
1537 static int
1538 instr_rx_translate(struct rte_swx_pipeline *p,
1539                    struct action *action,
1540                    char **tokens,
1541                    int n_tokens,
1542                    struct instruction *instr,
1543                    struct instruction_data *data __rte_unused)
1544 {
1545         struct field *f;
1546
1547         CHECK(!action, EINVAL);
1548         CHECK(n_tokens == 2, EINVAL);
1549
1550         f = metadata_field_parse(p, tokens[1]);
1551         CHECK(f, EINVAL);
1552
1553         instr->type = INSTR_RX;
1554         instr->io.io.offset = f->offset / 8;
1555         instr->io.io.n_bits = f->n_bits;
1556         return 0;
1557 }
1558
1559 /*
1560  * tx.
1561  */
1562 static int
1563 instr_tx_translate(struct rte_swx_pipeline *p,
1564                    struct action *action __rte_unused,
1565                    char **tokens,
1566                    int n_tokens,
1567                    struct instruction *instr,
1568                    struct instruction_data *data __rte_unused)
1569 {
1570         char *port = tokens[1];
1571         struct field *f;
1572         uint32_t port_val;
1573
1574         CHECK(n_tokens == 2, EINVAL);
1575
1576         f = metadata_field_parse(p, port);
1577         if (f) {
1578                 instr->type = INSTR_TX;
1579                 instr->io.io.offset = f->offset / 8;
1580                 instr->io.io.n_bits = f->n_bits;
1581                 return 0;
1582         }
1583
1584         /* TX_I. */
1585         port_val = strtoul(port, &port, 0);
1586         CHECK(!port[0], EINVAL);
1587
1588         instr->type = INSTR_TX_I;
1589         instr->io.io.val = port_val;
1590         return 0;
1591 }
1592
1593 static int
1594 instr_drop_translate(struct rte_swx_pipeline *p,
1595                      struct action *action __rte_unused,
1596                      char **tokens __rte_unused,
1597                      int n_tokens,
1598                      struct instruction *instr,
1599                      struct instruction_data *data __rte_unused)
1600 {
1601         CHECK(n_tokens == 1, EINVAL);
1602
1603         /* TX_I. */
1604         instr->type = INSTR_TX_I;
1605         instr->io.io.val = p->n_ports_out - 1;
1606         return 0;
1607 }
1608
1609 static inline void
1610 instr_tx_exec(struct rte_swx_pipeline *p)
1611 {
1612         struct thread *t = &p->threads[p->thread_id];
1613         struct instruction *ip = t->ip;
1614
1615         __instr_tx_exec(p, t, ip);
1616
1617         /* Thread. */
1618         thread_ip_reset(p, t);
1619         instr_rx_exec(p);
1620 }
1621
1622 static inline void
1623 instr_tx_i_exec(struct rte_swx_pipeline *p)
1624 {
1625         struct thread *t = &p->threads[p->thread_id];
1626         struct instruction *ip = t->ip;
1627
1628         __instr_tx_i_exec(p, t, ip);
1629
1630         /* Thread. */
1631         thread_ip_reset(p, t);
1632         instr_rx_exec(p);
1633 }
1634
1635 /*
1636  * extract.
1637  */
1638 static int
1639 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1640                             struct action *action,
1641                             char **tokens,
1642                             int n_tokens,
1643                             struct instruction *instr,
1644                             struct instruction_data *data __rte_unused)
1645 {
1646         struct header *h;
1647
1648         CHECK(!action, EINVAL);
1649         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1650
1651         h = header_parse(p, tokens[1]);
1652         CHECK(h, EINVAL);
1653
1654         if (n_tokens == 2) {
1655                 CHECK(!h->st->var_size, EINVAL);
1656
1657                 instr->type = INSTR_HDR_EXTRACT;
1658                 instr->io.hdr.header_id[0] = h->id;
1659                 instr->io.hdr.struct_id[0] = h->struct_id;
1660                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1661         } else {
1662                 struct field *mf;
1663
1664                 CHECK(h->st->var_size, EINVAL);
1665
1666                 mf = metadata_field_parse(p, tokens[2]);
1667                 CHECK(mf, EINVAL);
1668                 CHECK(!mf->var_size, EINVAL);
1669
1670                 instr->type = INSTR_HDR_EXTRACT_M;
1671                 instr->io.io.offset = mf->offset / 8;
1672                 instr->io.io.n_bits = mf->n_bits;
1673                 instr->io.hdr.header_id[0] = h->id;
1674                 instr->io.hdr.struct_id[0] = h->struct_id;
1675                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1676         }
1677
1678         return 0;
1679 }
1680
1681 static int
1682 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1683                               struct action *action,
1684                               char **tokens,
1685                               int n_tokens,
1686                               struct instruction *instr,
1687                               struct instruction_data *data __rte_unused)
1688 {
1689         struct header *h;
1690
1691         CHECK(!action, EINVAL);
1692         CHECK(n_tokens == 2, EINVAL);
1693
1694         h = header_parse(p, tokens[1]);
1695         CHECK(h, EINVAL);
1696         CHECK(!h->st->var_size, EINVAL);
1697
1698         instr->type = INSTR_HDR_LOOKAHEAD;
1699         instr->io.hdr.header_id[0] = h->id;
1700         instr->io.hdr.struct_id[0] = h->struct_id;
1701         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1702
1703         return 0;
1704 }
1705
1706 static inline void
1707 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1708 {
1709         struct thread *t = &p->threads[p->thread_id];
1710         struct instruction *ip = t->ip;
1711
1712         __instr_hdr_extract_exec(p, t, ip);
1713
1714         /* Thread. */
1715         thread_ip_inc(p);
1716 }
1717
1718 static inline void
1719 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1720 {
1721         struct thread *t = &p->threads[p->thread_id];
1722         struct instruction *ip = t->ip;
1723
1724         __instr_hdr_extract2_exec(p, t, ip);
1725
1726         /* Thread. */
1727         thread_ip_inc(p);
1728 }
1729
1730 static inline void
1731 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1732 {
1733         struct thread *t = &p->threads[p->thread_id];
1734         struct instruction *ip = t->ip;
1735
1736         __instr_hdr_extract3_exec(p, t, ip);
1737
1738         /* Thread. */
1739         thread_ip_inc(p);
1740 }
1741
1742 static inline void
1743 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1744 {
1745         struct thread *t = &p->threads[p->thread_id];
1746         struct instruction *ip = t->ip;
1747
1748         __instr_hdr_extract4_exec(p, t, ip);
1749
1750         /* Thread. */
1751         thread_ip_inc(p);
1752 }
1753
1754 static inline void
1755 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1756 {
1757         struct thread *t = &p->threads[p->thread_id];
1758         struct instruction *ip = t->ip;
1759
1760         __instr_hdr_extract5_exec(p, t, ip);
1761
1762         /* Thread. */
1763         thread_ip_inc(p);
1764 }
1765
1766 static inline void
1767 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1768 {
1769         struct thread *t = &p->threads[p->thread_id];
1770         struct instruction *ip = t->ip;
1771
1772         __instr_hdr_extract6_exec(p, t, ip);
1773
1774         /* Thread. */
1775         thread_ip_inc(p);
1776 }
1777
1778 static inline void
1779 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1780 {
1781         struct thread *t = &p->threads[p->thread_id];
1782         struct instruction *ip = t->ip;
1783
1784         __instr_hdr_extract7_exec(p, t, ip);
1785
1786         /* Thread. */
1787         thread_ip_inc(p);
1788 }
1789
1790 static inline void
1791 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1792 {
1793         struct thread *t = &p->threads[p->thread_id];
1794         struct instruction *ip = t->ip;
1795
1796         __instr_hdr_extract8_exec(p, t, ip);
1797
1798         /* Thread. */
1799         thread_ip_inc(p);
1800 }
1801
1802 static inline void
1803 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1804 {
1805         struct thread *t = &p->threads[p->thread_id];
1806         struct instruction *ip = t->ip;
1807
1808         __instr_hdr_extract_m_exec(p, t, ip);
1809
1810         /* Thread. */
1811         thread_ip_inc(p);
1812 }
1813
1814 static inline void
1815 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1816 {
1817         struct thread *t = &p->threads[p->thread_id];
1818         struct instruction *ip = t->ip;
1819
1820         __instr_hdr_lookahead_exec(p, t, ip);
1821
1822         /* Thread. */
1823         thread_ip_inc(p);
1824 }
1825
1826 /*
1827  * emit.
1828  */
1829 static int
1830 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1831                          struct action *action __rte_unused,
1832                          char **tokens,
1833                          int n_tokens,
1834                          struct instruction *instr,
1835                          struct instruction_data *data __rte_unused)
1836 {
1837         struct header *h;
1838
1839         CHECK(n_tokens == 2, EINVAL);
1840
1841         h = header_parse(p, tokens[1]);
1842         CHECK(h, EINVAL);
1843
1844         instr->type = INSTR_HDR_EMIT;
1845         instr->io.hdr.header_id[0] = h->id;
1846         instr->io.hdr.struct_id[0] = h->struct_id;
1847         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1848         return 0;
1849 }
1850
1851 static inline void
1852 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1853 {
1854         struct thread *t = &p->threads[p->thread_id];
1855         struct instruction *ip = t->ip;
1856
1857         __instr_hdr_emit_exec(p, t, ip);
1858
1859         /* Thread. */
1860         thread_ip_inc(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit2_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit3_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit4_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit5_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 static inline void
1929 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1930 {
1931         struct thread *t = &p->threads[p->thread_id];
1932         struct instruction *ip = t->ip;
1933
1934         __instr_hdr_emit6_tx_exec(p, t, ip);
1935
1936         /* Thread. */
1937         thread_ip_reset(p, t);
1938         instr_rx_exec(p);
1939 }
1940
1941 static inline void
1942 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1943 {
1944         struct thread *t = &p->threads[p->thread_id];
1945         struct instruction *ip = t->ip;
1946
1947         __instr_hdr_emit7_tx_exec(p, t, ip);
1948
1949         /* Thread. */
1950         thread_ip_reset(p, t);
1951         instr_rx_exec(p);
1952 }
1953
1954 static inline void
1955 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1956 {
1957         struct thread *t = &p->threads[p->thread_id];
1958         struct instruction *ip = t->ip;
1959
1960         __instr_hdr_emit8_tx_exec(p, t, ip);
1961
1962         /* Thread. */
1963         thread_ip_reset(p, t);
1964         instr_rx_exec(p);
1965 }
1966
1967 /*
1968  * validate.
1969  */
1970 static int
1971 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1972                              struct action *action __rte_unused,
1973                              char **tokens,
1974                              int n_tokens,
1975                              struct instruction *instr,
1976                              struct instruction_data *data __rte_unused)
1977 {
1978         struct header *h;
1979
1980         CHECK(n_tokens == 2, EINVAL);
1981
1982         h = header_parse(p, tokens[1]);
1983         CHECK(h, EINVAL);
1984
1985         instr->type = INSTR_HDR_VALIDATE;
1986         instr->valid.header_id = h->id;
1987         return 0;
1988 }
1989
1990 static inline void
1991 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1992 {
1993         struct thread *t = &p->threads[p->thread_id];
1994         struct instruction *ip = t->ip;
1995
1996         __instr_hdr_validate_exec(p, t, ip);
1997
1998         /* Thread. */
1999         thread_ip_inc(p);
2000 }
2001
2002 /*
2003  * invalidate.
2004  */
2005 static int
2006 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2007                                struct action *action __rte_unused,
2008                                char **tokens,
2009                                int n_tokens,
2010                                struct instruction *instr,
2011                                struct instruction_data *data __rte_unused)
2012 {
2013         struct header *h;
2014
2015         CHECK(n_tokens == 2, EINVAL);
2016
2017         h = header_parse(p, tokens[1]);
2018         CHECK(h, EINVAL);
2019
2020         instr->type = INSTR_HDR_INVALIDATE;
2021         instr->valid.header_id = h->id;
2022         return 0;
2023 }
2024
2025 static inline void
2026 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2027 {
2028         struct thread *t = &p->threads[p->thread_id];
2029         struct instruction *ip = t->ip;
2030
2031         __instr_hdr_invalidate_exec(p, t, ip);
2032
2033         /* Thread. */
2034         thread_ip_inc(p);
2035 }
2036
2037 /*
2038  * table.
2039  */
2040 static struct table *
2041 table_find(struct rte_swx_pipeline *p, const char *name);
2042
2043 static struct selector *
2044 selector_find(struct rte_swx_pipeline *p, const char *name);
2045
2046 static struct learner *
2047 learner_find(struct rte_swx_pipeline *p, const char *name);
2048
2049 static int
2050 instr_table_translate(struct rte_swx_pipeline *p,
2051                       struct action *action,
2052                       char **tokens,
2053                       int n_tokens,
2054                       struct instruction *instr,
2055                       struct instruction_data *data __rte_unused)
2056 {
2057         struct table *t;
2058         struct selector *s;
2059         struct learner *l;
2060
2061         CHECK(!action, EINVAL);
2062         CHECK(n_tokens == 2, EINVAL);
2063
2064         t = table_find(p, tokens[1]);
2065         if (t) {
2066                 instr->type = INSTR_TABLE;
2067                 instr->table.table_id = t->id;
2068                 return 0;
2069         }
2070
2071         s = selector_find(p, tokens[1]);
2072         if (s) {
2073                 instr->type = INSTR_SELECTOR;
2074                 instr->table.table_id = s->id;
2075                 return 0;
2076         }
2077
2078         l = learner_find(p, tokens[1]);
2079         if (l) {
2080                 instr->type = INSTR_LEARNER;
2081                 instr->table.table_id = l->id;
2082                 return 0;
2083         }
2084
2085         CHECK(0, EINVAL);
2086 }
2087
2088 static inline void
2089 instr_table_exec(struct rte_swx_pipeline *p)
2090 {
2091         struct thread *t = &p->threads[p->thread_id];
2092         struct instruction *ip = t->ip;
2093         uint32_t table_id = ip->table.table_id;
2094         struct rte_swx_table_state *ts = &t->table_state[table_id];
2095         struct table_runtime *table = &t->tables[table_id];
2096         struct table_statistics *stats = &p->table_stats[table_id];
2097         uint64_t action_id, n_pkts_hit, n_pkts_action;
2098         uint8_t *action_data;
2099         int done, hit;
2100
2101         /* Table. */
2102         done = table->func(ts->obj,
2103                            table->mailbox,
2104                            table->key,
2105                            &action_id,
2106                            &action_data,
2107                            &hit);
2108         if (!done) {
2109                 /* Thread. */
2110                 TRACE("[Thread %2u] table %u (not finalized)\n",
2111                       p->thread_id,
2112                       table_id);
2113
2114                 thread_yield(p);
2115                 return;
2116         }
2117
2118         action_id = hit ? action_id : ts->default_action_id;
2119         action_data = hit ? action_data : ts->default_action_data;
2120         n_pkts_hit = stats->n_pkts_hit[hit];
2121         n_pkts_action = stats->n_pkts_action[action_id];
2122
2123         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2124               p->thread_id,
2125               table_id,
2126               hit ? "hit" : "miss",
2127               (uint32_t)action_id);
2128
2129         t->action_id = action_id;
2130         t->structs[0] = action_data;
2131         t->hit = hit;
2132         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2133         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2134
2135         /* Thread. */
2136         thread_ip_action_call(p, t, action_id);
2137 }
2138
2139 static inline void
2140 instr_table_af_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t table_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[table_id];
2146         struct table_runtime *table = &t->tables[table_id];
2147         struct table_statistics *stats = &p->table_stats[table_id];
2148         uint64_t action_id, n_pkts_hit, n_pkts_action;
2149         uint8_t *action_data;
2150         action_func_t action_func;
2151         int done, hit;
2152
2153         /* Table. */
2154         done = table->func(ts->obj,
2155                            table->mailbox,
2156                            table->key,
2157                            &action_id,
2158                            &action_data,
2159                            &hit);
2160         if (!done) {
2161                 /* Thread. */
2162                 TRACE("[Thread %2u] table %u (not finalized)\n",
2163                       p->thread_id,
2164                       table_id);
2165
2166                 thread_yield(p);
2167                 return;
2168         }
2169
2170         action_id = hit ? action_id : ts->default_action_id;
2171         action_data = hit ? action_data : ts->default_action_data;
2172         action_func = p->action_funcs[action_id];
2173         n_pkts_hit = stats->n_pkts_hit[hit];
2174         n_pkts_action = stats->n_pkts_action[action_id];
2175
2176         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2177               p->thread_id,
2178               table_id,
2179               hit ? "hit" : "miss",
2180               (uint32_t)action_id);
2181
2182         t->action_id = action_id;
2183         t->structs[0] = action_data;
2184         t->hit = hit;
2185         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2186         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2187
2188         /* Thread. */
2189         thread_ip_inc(p);
2190
2191         /* Action. */
2192         action_func(p);
2193 }
2194
2195 static inline void
2196 instr_selector_exec(struct rte_swx_pipeline *p)
2197 {
2198         struct thread *t = &p->threads[p->thread_id];
2199         struct instruction *ip = t->ip;
2200         uint32_t selector_id = ip->table.table_id;
2201         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2202         struct selector_runtime *selector = &t->selectors[selector_id];
2203         struct selector_statistics *stats = &p->selector_stats[selector_id];
2204         uint64_t n_pkts = stats->n_pkts;
2205         int done;
2206
2207         /* Table. */
2208         done = rte_swx_table_selector_select(ts->obj,
2209                            selector->mailbox,
2210                            selector->group_id_buffer,
2211                            selector->selector_buffer,
2212                            selector->member_id_buffer);
2213         if (!done) {
2214                 /* Thread. */
2215                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2216                       p->thread_id,
2217                       selector_id);
2218
2219                 thread_yield(p);
2220                 return;
2221         }
2222
2223
2224         TRACE("[Thread %2u] selector %u\n",
2225               p->thread_id,
2226               selector_id);
2227
2228         stats->n_pkts = n_pkts + 1;
2229
2230         /* Thread. */
2231         thread_ip_inc(p);
2232 }
2233
2234 static inline void
2235 instr_learner_exec(struct rte_swx_pipeline *p)
2236 {
2237         struct thread *t = &p->threads[p->thread_id];
2238         struct instruction *ip = t->ip;
2239         uint32_t learner_id = ip->table.table_id;
2240         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2241                 p->n_selectors + learner_id];
2242         struct learner_runtime *l = &t->learners[learner_id];
2243         struct learner_statistics *stats = &p->learner_stats[learner_id];
2244         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2245         uint8_t *action_data;
2246         int done, hit;
2247
2248         /* Table. */
2249         time = rte_get_tsc_cycles();
2250
2251         done = rte_swx_table_learner_lookup(ts->obj,
2252                                             l->mailbox,
2253                                             time,
2254                                             l->key,
2255                                             &action_id,
2256                                             &action_data,
2257                                             &hit);
2258         if (!done) {
2259                 /* Thread. */
2260                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2261                       p->thread_id,
2262                       learner_id);
2263
2264                 thread_yield(p);
2265                 return;
2266         }
2267
2268         action_id = hit ? action_id : ts->default_action_id;
2269         action_data = hit ? action_data : ts->default_action_data;
2270         n_pkts_hit = stats->n_pkts_hit[hit];
2271         n_pkts_action = stats->n_pkts_action[action_id];
2272
2273         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2274               p->thread_id,
2275               learner_id,
2276               hit ? "hit" : "miss",
2277               (uint32_t)action_id);
2278
2279         t->action_id = action_id;
2280         t->structs[0] = action_data;
2281         t->hit = hit;
2282         t->learner_id = learner_id;
2283         t->time = time;
2284         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2285         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2286
2287         /* Thread. */
2288         thread_ip_action_call(p, t, action_id);
2289 }
2290
2291 static inline void
2292 instr_learner_af_exec(struct rte_swx_pipeline *p)
2293 {
2294         struct thread *t = &p->threads[p->thread_id];
2295         struct instruction *ip = t->ip;
2296         uint32_t learner_id = ip->table.table_id;
2297         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2298                 p->n_selectors + learner_id];
2299         struct learner_runtime *l = &t->learners[learner_id];
2300         struct learner_statistics *stats = &p->learner_stats[learner_id];
2301         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2302         uint8_t *action_data;
2303         action_func_t action_func;
2304         int done, hit;
2305
2306         /* Table. */
2307         time = rte_get_tsc_cycles();
2308
2309         done = rte_swx_table_learner_lookup(ts->obj,
2310                                             l->mailbox,
2311                                             time,
2312                                             l->key,
2313                                             &action_id,
2314                                             &action_data,
2315                                             &hit);
2316         if (!done) {
2317                 /* Thread. */
2318                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2319                       p->thread_id,
2320                       learner_id);
2321
2322                 thread_yield(p);
2323                 return;
2324         }
2325
2326         action_id = hit ? action_id : ts->default_action_id;
2327         action_data = hit ? action_data : ts->default_action_data;
2328         action_func = p->action_funcs[action_id];
2329         n_pkts_hit = stats->n_pkts_hit[hit];
2330         n_pkts_action = stats->n_pkts_action[action_id];
2331
2332         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2333               p->thread_id,
2334               learner_id,
2335               hit ? "hit" : "miss",
2336               (uint32_t)action_id);
2337
2338         t->action_id = action_id;
2339         t->structs[0] = action_data;
2340         t->hit = hit;
2341         t->learner_id = learner_id;
2342         t->time = time;
2343         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2344         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2345
2346         /* Thread. */
2347         thread_ip_action_call(p, t, action_id);
2348
2349         /* Action */
2350         action_func(p);
2351 }
2352
2353 /*
2354  * learn.
2355  */
2356 static struct action *
2357 action_find(struct rte_swx_pipeline *p, const char *name);
2358
2359 static int
2360 action_has_nbo_args(struct action *a);
2361
2362 static int
2363 instr_learn_translate(struct rte_swx_pipeline *p,
2364                       struct action *action,
2365                       char **tokens,
2366                       int n_tokens,
2367                       struct instruction *instr,
2368                       struct instruction_data *data __rte_unused)
2369 {
2370         struct action *a;
2371
2372         CHECK(action, EINVAL);
2373         CHECK(n_tokens == 2, EINVAL);
2374
2375         a = action_find(p, tokens[1]);
2376         CHECK(a, EINVAL);
2377         CHECK(!action_has_nbo_args(a), EINVAL);
2378
2379         instr->type = INSTR_LEARNER_LEARN;
2380         instr->learn.action_id = a->id;
2381
2382         return 0;
2383 }
2384
2385 static inline void
2386 instr_learn_exec(struct rte_swx_pipeline *p)
2387 {
2388         struct thread *t = &p->threads[p->thread_id];
2389         struct instruction *ip = t->ip;
2390
2391         __instr_learn_exec(p, t, ip);
2392
2393         /* Thread. */
2394         thread_ip_inc(p);
2395 }
2396
2397 /*
2398  * forget.
2399  */
2400 static int
2401 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2402                        struct action *action,
2403                        char **tokens __rte_unused,
2404                        int n_tokens,
2405                        struct instruction *instr,
2406                        struct instruction_data *data __rte_unused)
2407 {
2408         CHECK(action, EINVAL);
2409         CHECK(n_tokens == 1, EINVAL);
2410
2411         instr->type = INSTR_LEARNER_FORGET;
2412
2413         return 0;
2414 }
2415
2416 static inline void
2417 instr_forget_exec(struct rte_swx_pipeline *p)
2418 {
2419         struct thread *t = &p->threads[p->thread_id];
2420         struct instruction *ip = t->ip;
2421
2422         __instr_forget_exec(p, t, ip);
2423
2424         /* Thread. */
2425         thread_ip_inc(p);
2426 }
2427
2428 /*
2429  * extern.
2430  */
2431 static int
2432 instr_extern_translate(struct rte_swx_pipeline *p,
2433                        struct action *action __rte_unused,
2434                        char **tokens,
2435                        int n_tokens,
2436                        struct instruction *instr,
2437                        struct instruction_data *data __rte_unused)
2438 {
2439         char *token = tokens[1];
2440
2441         CHECK(n_tokens == 2, EINVAL);
2442
2443         if (token[0] == 'e') {
2444                 struct extern_obj *obj;
2445                 struct extern_type_member_func *func;
2446
2447                 func = extern_obj_member_func_parse(p, token, &obj);
2448                 CHECK(func, EINVAL);
2449
2450                 instr->type = INSTR_EXTERN_OBJ;
2451                 instr->ext_obj.ext_obj_id = obj->id;
2452                 instr->ext_obj.func_id = func->id;
2453
2454                 return 0;
2455         }
2456
2457         if (token[0] == 'f') {
2458                 struct extern_func *func;
2459
2460                 func = extern_func_parse(p, token);
2461                 CHECK(func, EINVAL);
2462
2463                 instr->type = INSTR_EXTERN_FUNC;
2464                 instr->ext_func.ext_func_id = func->id;
2465
2466                 return 0;
2467         }
2468
2469         CHECK(0, EINVAL);
2470 }
2471
2472 static inline void
2473 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2474 {
2475         struct thread *t = &p->threads[p->thread_id];
2476         struct instruction *ip = t->ip;
2477         uint32_t done;
2478
2479         /* Extern object member function execute. */
2480         done = __instr_extern_obj_exec(p, t, ip);
2481
2482         /* Thread. */
2483         thread_ip_inc_cond(t, done);
2484         thread_yield_cond(p, done ^ 1);
2485 }
2486
2487 static inline void
2488 instr_extern_func_exec(struct rte_swx_pipeline *p)
2489 {
2490         struct thread *t = &p->threads[p->thread_id];
2491         struct instruction *ip = t->ip;
2492         uint32_t done;
2493
2494         /* Extern function execute. */
2495         done = __instr_extern_func_exec(p, t, ip);
2496
2497         /* Thread. */
2498         thread_ip_inc_cond(t, done);
2499         thread_yield_cond(p, done ^ 1);
2500 }
2501
2502 /*
2503  * mov.
2504  */
2505 static int
2506 instr_mov_translate(struct rte_swx_pipeline *p,
2507                     struct action *action,
2508                     char **tokens,
2509                     int n_tokens,
2510                     struct instruction *instr,
2511                     struct instruction_data *data __rte_unused)
2512 {
2513         char *dst = tokens[1], *src = tokens[2];
2514         struct field *fdst, *fsrc;
2515         uint64_t src_val;
2516         uint32_t dst_struct_id = 0, src_struct_id = 0;
2517
2518         CHECK(n_tokens == 3, EINVAL);
2519
2520         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2521         CHECK(fdst, EINVAL);
2522         CHECK(!fdst->var_size, EINVAL);
2523
2524         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2525         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2526         if (fsrc) {
2527                 CHECK(!fsrc->var_size, EINVAL);
2528
2529                 instr->type = INSTR_MOV;
2530                 if (dst[0] != 'h' && src[0] == 'h')
2531                         instr->type = INSTR_MOV_MH;
2532                 if (dst[0] == 'h' && src[0] != 'h')
2533                         instr->type = INSTR_MOV_HM;
2534                 if (dst[0] == 'h' && src[0] == 'h')
2535                         instr->type = INSTR_MOV_HH;
2536
2537                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2538                 instr->mov.dst.n_bits = fdst->n_bits;
2539                 instr->mov.dst.offset = fdst->offset / 8;
2540                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2541                 instr->mov.src.n_bits = fsrc->n_bits;
2542                 instr->mov.src.offset = fsrc->offset / 8;
2543                 return 0;
2544         }
2545
2546         /* MOV_I. */
2547         src_val = strtoull(src, &src, 0);
2548         CHECK(!src[0], EINVAL);
2549
2550         if (dst[0] == 'h')
2551                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2552
2553         instr->type = INSTR_MOV_I;
2554         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2555         instr->mov.dst.n_bits = fdst->n_bits;
2556         instr->mov.dst.offset = fdst->offset / 8;
2557         instr->mov.src_val = src_val;
2558         return 0;
2559 }
2560
2561 static inline void
2562 instr_mov_exec(struct rte_swx_pipeline *p)
2563 {
2564         struct thread *t = &p->threads[p->thread_id];
2565         struct instruction *ip = t->ip;
2566
2567         __instr_mov_exec(p, t, ip);
2568
2569         /* Thread. */
2570         thread_ip_inc(p);
2571 }
2572
2573 static inline void
2574 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2575 {
2576         struct thread *t = &p->threads[p->thread_id];
2577         struct instruction *ip = t->ip;
2578
2579         __instr_mov_mh_exec(p, t, ip);
2580
2581         /* Thread. */
2582         thread_ip_inc(p);
2583 }
2584
2585 static inline void
2586 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2587 {
2588         struct thread *t = &p->threads[p->thread_id];
2589         struct instruction *ip = t->ip;
2590
2591         __instr_mov_hm_exec(p, t, ip);
2592
2593         /* Thread. */
2594         thread_ip_inc(p);
2595 }
2596
2597 static inline void
2598 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2599 {
2600         struct thread *t = &p->threads[p->thread_id];
2601         struct instruction *ip = t->ip;
2602
2603         __instr_mov_hh_exec(p, t, ip);
2604
2605         /* Thread. */
2606         thread_ip_inc(p);
2607 }
2608
2609 static inline void
2610 instr_mov_i_exec(struct rte_swx_pipeline *p)
2611 {
2612         struct thread *t = &p->threads[p->thread_id];
2613         struct instruction *ip = t->ip;
2614
2615         __instr_mov_i_exec(p, t, ip);
2616
2617         /* Thread. */
2618         thread_ip_inc(p);
2619 }
2620
2621 /*
2622  * dma.
2623  */
2624 static inline void
2625 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2626 {
2627         struct thread *t = &p->threads[p->thread_id];
2628         struct instruction *ip = t->ip;
2629
2630         __instr_dma_ht_exec(p, t, ip);
2631
2632         /* Thread. */
2633         thread_ip_inc(p);
2634 }
2635
2636 static inline void
2637 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2638 {
2639         struct thread *t = &p->threads[p->thread_id];
2640         struct instruction *ip = t->ip;
2641
2642         __instr_dma_ht2_exec(p, t, ip);
2643
2644         /* Thread. */
2645         thread_ip_inc(p);
2646 }
2647
2648 static inline void
2649 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2650 {
2651         struct thread *t = &p->threads[p->thread_id];
2652         struct instruction *ip = t->ip;
2653
2654         __instr_dma_ht3_exec(p, t, ip);
2655
2656         /* Thread. */
2657         thread_ip_inc(p);
2658 }
2659
2660 static inline void
2661 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2662 {
2663         struct thread *t = &p->threads[p->thread_id];
2664         struct instruction *ip = t->ip;
2665
2666         __instr_dma_ht4_exec(p, t, ip);
2667
2668         /* Thread. */
2669         thread_ip_inc(p);
2670 }
2671
2672 static inline void
2673 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2674 {
2675         struct thread *t = &p->threads[p->thread_id];
2676         struct instruction *ip = t->ip;
2677
2678         __instr_dma_ht5_exec(p, t, ip);
2679
2680         /* Thread. */
2681         thread_ip_inc(p);
2682 }
2683
2684 static inline void
2685 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2686 {
2687         struct thread *t = &p->threads[p->thread_id];
2688         struct instruction *ip = t->ip;
2689
2690         __instr_dma_ht6_exec(p, t, ip);
2691
2692         /* Thread. */
2693         thread_ip_inc(p);
2694 }
2695
2696 static inline void
2697 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2698 {
2699         struct thread *t = &p->threads[p->thread_id];
2700         struct instruction *ip = t->ip;
2701
2702         __instr_dma_ht7_exec(p, t, ip);
2703
2704         /* Thread. */
2705         thread_ip_inc(p);
2706 }
2707
2708 static inline void
2709 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2710 {
2711         struct thread *t = &p->threads[p->thread_id];
2712         struct instruction *ip = t->ip;
2713
2714         __instr_dma_ht8_exec(p, t, ip);
2715
2716         /* Thread. */
2717         thread_ip_inc(p);
2718 }
2719
2720 /*
2721  * alu.
2722  */
2723 static int
2724 instr_alu_add_translate(struct rte_swx_pipeline *p,
2725                         struct action *action,
2726                         char **tokens,
2727                         int n_tokens,
2728                         struct instruction *instr,
2729                         struct instruction_data *data __rte_unused)
2730 {
2731         char *dst = tokens[1], *src = tokens[2];
2732         struct field *fdst, *fsrc;
2733         uint64_t src_val;
2734         uint32_t dst_struct_id = 0, src_struct_id = 0;
2735
2736         CHECK(n_tokens == 3, EINVAL);
2737
2738         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2739         CHECK(fdst, EINVAL);
2740         CHECK(!fdst->var_size, EINVAL);
2741
2742         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2743         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2744         if (fsrc) {
2745                 CHECK(!fsrc->var_size, EINVAL);
2746
2747                 instr->type = INSTR_ALU_ADD;
2748                 if (dst[0] == 'h' && src[0] != 'h')
2749                         instr->type = INSTR_ALU_ADD_HM;
2750                 if (dst[0] != 'h' && src[0] == 'h')
2751                         instr->type = INSTR_ALU_ADD_MH;
2752                 if (dst[0] == 'h' && src[0] == 'h')
2753                         instr->type = INSTR_ALU_ADD_HH;
2754
2755                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2756                 instr->alu.dst.n_bits = fdst->n_bits;
2757                 instr->alu.dst.offset = fdst->offset / 8;
2758                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2759                 instr->alu.src.n_bits = fsrc->n_bits;
2760                 instr->alu.src.offset = fsrc->offset / 8;
2761                 return 0;
2762         }
2763
2764         /* ADD_MI, ADD_HI. */
2765         src_val = strtoull(src, &src, 0);
2766         CHECK(!src[0], EINVAL);
2767
2768         instr->type = INSTR_ALU_ADD_MI;
2769         if (dst[0] == 'h')
2770                 instr->type = INSTR_ALU_ADD_HI;
2771
2772         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2773         instr->alu.dst.n_bits = fdst->n_bits;
2774         instr->alu.dst.offset = fdst->offset / 8;
2775         instr->alu.src_val = src_val;
2776         return 0;
2777 }
2778
2779 static int
2780 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2781                         struct action *action,
2782                         char **tokens,
2783                         int n_tokens,
2784                         struct instruction *instr,
2785                         struct instruction_data *data __rte_unused)
2786 {
2787         char *dst = tokens[1], *src = tokens[2];
2788         struct field *fdst, *fsrc;
2789         uint64_t src_val;
2790         uint32_t dst_struct_id = 0, src_struct_id = 0;
2791
2792         CHECK(n_tokens == 3, EINVAL);
2793
2794         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2795         CHECK(fdst, EINVAL);
2796         CHECK(!fdst->var_size, EINVAL);
2797
2798         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2799         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2800         if (fsrc) {
2801                 CHECK(!fsrc->var_size, EINVAL);
2802
2803                 instr->type = INSTR_ALU_SUB;
2804                 if (dst[0] == 'h' && src[0] != 'h')
2805                         instr->type = INSTR_ALU_SUB_HM;
2806                 if (dst[0] != 'h' && src[0] == 'h')
2807                         instr->type = INSTR_ALU_SUB_MH;
2808                 if (dst[0] == 'h' && src[0] == 'h')
2809                         instr->type = INSTR_ALU_SUB_HH;
2810
2811                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2812                 instr->alu.dst.n_bits = fdst->n_bits;
2813                 instr->alu.dst.offset = fdst->offset / 8;
2814                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2815                 instr->alu.src.n_bits = fsrc->n_bits;
2816                 instr->alu.src.offset = fsrc->offset / 8;
2817                 return 0;
2818         }
2819
2820         /* SUB_MI, SUB_HI. */
2821         src_val = strtoull(src, &src, 0);
2822         CHECK(!src[0], EINVAL);
2823
2824         instr->type = INSTR_ALU_SUB_MI;
2825         if (dst[0] == 'h')
2826                 instr->type = INSTR_ALU_SUB_HI;
2827
2828         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2829         instr->alu.dst.n_bits = fdst->n_bits;
2830         instr->alu.dst.offset = fdst->offset / 8;
2831         instr->alu.src_val = src_val;
2832         return 0;
2833 }
2834
2835 static int
2836 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2837                           struct action *action __rte_unused,
2838                           char **tokens,
2839                           int n_tokens,
2840                           struct instruction *instr,
2841                           struct instruction_data *data __rte_unused)
2842 {
2843         char *dst = tokens[1], *src = tokens[2];
2844         struct header *hdst, *hsrc;
2845         struct field *fdst, *fsrc;
2846
2847         CHECK(n_tokens == 3, EINVAL);
2848
2849         fdst = header_field_parse(p, dst, &hdst);
2850         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2851         CHECK(!fdst->var_size, EINVAL);
2852
2853         /* CKADD_FIELD. */
2854         fsrc = header_field_parse(p, src, &hsrc);
2855         if (fsrc) {
2856                 CHECK(!fsrc->var_size, EINVAL);
2857
2858                 instr->type = INSTR_ALU_CKADD_FIELD;
2859                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2860                 instr->alu.dst.n_bits = fdst->n_bits;
2861                 instr->alu.dst.offset = fdst->offset / 8;
2862                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2863                 instr->alu.src.n_bits = fsrc->n_bits;
2864                 instr->alu.src.offset = fsrc->offset / 8;
2865                 return 0;
2866         }
2867
2868         /* CKADD_STRUCT, CKADD_STRUCT20. */
2869         hsrc = header_parse(p, src);
2870         CHECK(hsrc, EINVAL);
2871         CHECK(!hsrc->st->var_size, EINVAL);
2872
2873         instr->type = INSTR_ALU_CKADD_STRUCT;
2874         if ((hsrc->st->n_bits / 8) == 20)
2875                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2876
2877         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2878         instr->alu.dst.n_bits = fdst->n_bits;
2879         instr->alu.dst.offset = fdst->offset / 8;
2880         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2881         instr->alu.src.n_bits = hsrc->st->n_bits;
2882         instr->alu.src.offset = 0; /* Unused. */
2883         return 0;
2884 }
2885
2886 static int
2887 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2888                           struct action *action __rte_unused,
2889                           char **tokens,
2890                           int n_tokens,
2891                           struct instruction *instr,
2892                           struct instruction_data *data __rte_unused)
2893 {
2894         char *dst = tokens[1], *src = tokens[2];
2895         struct header *hdst, *hsrc;
2896         struct field *fdst, *fsrc;
2897
2898         CHECK(n_tokens == 3, EINVAL);
2899
2900         fdst = header_field_parse(p, dst, &hdst);
2901         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2902         CHECK(!fdst->var_size, EINVAL);
2903
2904         fsrc = header_field_parse(p, src, &hsrc);
2905         CHECK(fsrc, EINVAL);
2906         CHECK(!fsrc->var_size, EINVAL);
2907
2908         instr->type = INSTR_ALU_CKSUB_FIELD;
2909         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2910         instr->alu.dst.n_bits = fdst->n_bits;
2911         instr->alu.dst.offset = fdst->offset / 8;
2912         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2913         instr->alu.src.n_bits = fsrc->n_bits;
2914         instr->alu.src.offset = fsrc->offset / 8;
2915         return 0;
2916 }
2917
2918 static int
2919 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2920                         struct action *action,
2921                         char **tokens,
2922                         int n_tokens,
2923                         struct instruction *instr,
2924                         struct instruction_data *data __rte_unused)
2925 {
2926         char *dst = tokens[1], *src = tokens[2];
2927         struct field *fdst, *fsrc;
2928         uint64_t src_val;
2929         uint32_t dst_struct_id = 0, src_struct_id = 0;
2930
2931         CHECK(n_tokens == 3, EINVAL);
2932
2933         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2934         CHECK(fdst, EINVAL);
2935         CHECK(!fdst->var_size, EINVAL);
2936
2937         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2938         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2939         if (fsrc) {
2940                 CHECK(!fsrc->var_size, EINVAL);
2941
2942                 instr->type = INSTR_ALU_SHL;
2943                 if (dst[0] == 'h' && src[0] != 'h')
2944                         instr->type = INSTR_ALU_SHL_HM;
2945                 if (dst[0] != 'h' && src[0] == 'h')
2946                         instr->type = INSTR_ALU_SHL_MH;
2947                 if (dst[0] == 'h' && src[0] == 'h')
2948                         instr->type = INSTR_ALU_SHL_HH;
2949
2950                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2951                 instr->alu.dst.n_bits = fdst->n_bits;
2952                 instr->alu.dst.offset = fdst->offset / 8;
2953                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2954                 instr->alu.src.n_bits = fsrc->n_bits;
2955                 instr->alu.src.offset = fsrc->offset / 8;
2956                 return 0;
2957         }
2958
2959         /* SHL_MI, SHL_HI. */
2960         src_val = strtoull(src, &src, 0);
2961         CHECK(!src[0], EINVAL);
2962
2963         instr->type = INSTR_ALU_SHL_MI;
2964         if (dst[0] == 'h')
2965                 instr->type = INSTR_ALU_SHL_HI;
2966
2967         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2968         instr->alu.dst.n_bits = fdst->n_bits;
2969         instr->alu.dst.offset = fdst->offset / 8;
2970         instr->alu.src_val = src_val;
2971         return 0;
2972 }
2973
2974 static int
2975 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2976                         struct action *action,
2977                         char **tokens,
2978                         int n_tokens,
2979                         struct instruction *instr,
2980                         struct instruction_data *data __rte_unused)
2981 {
2982         char *dst = tokens[1], *src = tokens[2];
2983         struct field *fdst, *fsrc;
2984         uint64_t src_val;
2985         uint32_t dst_struct_id = 0, src_struct_id = 0;
2986
2987         CHECK(n_tokens == 3, EINVAL);
2988
2989         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2990         CHECK(fdst, EINVAL);
2991         CHECK(!fdst->var_size, EINVAL);
2992
2993         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2994         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2995         if (fsrc) {
2996                 CHECK(!fsrc->var_size, EINVAL);
2997
2998                 instr->type = INSTR_ALU_SHR;
2999                 if (dst[0] == 'h' && src[0] != 'h')
3000                         instr->type = INSTR_ALU_SHR_HM;
3001                 if (dst[0] != 'h' && src[0] == 'h')
3002                         instr->type = INSTR_ALU_SHR_MH;
3003                 if (dst[0] == 'h' && src[0] == 'h')
3004                         instr->type = INSTR_ALU_SHR_HH;
3005
3006                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3007                 instr->alu.dst.n_bits = fdst->n_bits;
3008                 instr->alu.dst.offset = fdst->offset / 8;
3009                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3010                 instr->alu.src.n_bits = fsrc->n_bits;
3011                 instr->alu.src.offset = fsrc->offset / 8;
3012                 return 0;
3013         }
3014
3015         /* SHR_MI, SHR_HI. */
3016         src_val = strtoull(src, &src, 0);
3017         CHECK(!src[0], EINVAL);
3018
3019         instr->type = INSTR_ALU_SHR_MI;
3020         if (dst[0] == 'h')
3021                 instr->type = INSTR_ALU_SHR_HI;
3022
3023         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3024         instr->alu.dst.n_bits = fdst->n_bits;
3025         instr->alu.dst.offset = fdst->offset / 8;
3026         instr->alu.src_val = src_val;
3027         return 0;
3028 }
3029
3030 static int
3031 instr_alu_and_translate(struct rte_swx_pipeline *p,
3032                         struct action *action,
3033                         char **tokens,
3034                         int n_tokens,
3035                         struct instruction *instr,
3036                         struct instruction_data *data __rte_unused)
3037 {
3038         char *dst = tokens[1], *src = tokens[2];
3039         struct field *fdst, *fsrc;
3040         uint64_t src_val;
3041         uint32_t dst_struct_id = 0, src_struct_id = 0;
3042
3043         CHECK(n_tokens == 3, EINVAL);
3044
3045         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3046         CHECK(fdst, EINVAL);
3047         CHECK(!fdst->var_size, EINVAL);
3048
3049         /* AND, AND_MH, AND_HM, AND_HH. */
3050         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3051         if (fsrc) {
3052                 CHECK(!fsrc->var_size, EINVAL);
3053
3054                 instr->type = INSTR_ALU_AND;
3055                 if (dst[0] != 'h' && src[0] == 'h')
3056                         instr->type = INSTR_ALU_AND_MH;
3057                 if (dst[0] == 'h' && src[0] != 'h')
3058                         instr->type = INSTR_ALU_AND_HM;
3059                 if (dst[0] == 'h' && src[0] == 'h')
3060                         instr->type = INSTR_ALU_AND_HH;
3061
3062                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3063                 instr->alu.dst.n_bits = fdst->n_bits;
3064                 instr->alu.dst.offset = fdst->offset / 8;
3065                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3066                 instr->alu.src.n_bits = fsrc->n_bits;
3067                 instr->alu.src.offset = fsrc->offset / 8;
3068                 return 0;
3069         }
3070
3071         /* AND_I. */
3072         src_val = strtoull(src, &src, 0);
3073         CHECK(!src[0], EINVAL);
3074
3075         if (dst[0] == 'h')
3076                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3077
3078         instr->type = INSTR_ALU_AND_I;
3079         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3080         instr->alu.dst.n_bits = fdst->n_bits;
3081         instr->alu.dst.offset = fdst->offset / 8;
3082         instr->alu.src_val = src_val;
3083         return 0;
3084 }
3085
3086 static int
3087 instr_alu_or_translate(struct rte_swx_pipeline *p,
3088                        struct action *action,
3089                        char **tokens,
3090                        int n_tokens,
3091                        struct instruction *instr,
3092                        struct instruction_data *data __rte_unused)
3093 {
3094         char *dst = tokens[1], *src = tokens[2];
3095         struct field *fdst, *fsrc;
3096         uint64_t src_val;
3097         uint32_t dst_struct_id = 0, src_struct_id = 0;
3098
3099         CHECK(n_tokens == 3, EINVAL);
3100
3101         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3102         CHECK(fdst, EINVAL);
3103         CHECK(!fdst->var_size, EINVAL);
3104
3105         /* OR, OR_MH, OR_HM, OR_HH. */
3106         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3107         if (fsrc) {
3108                 CHECK(!fsrc->var_size, EINVAL);
3109
3110                 instr->type = INSTR_ALU_OR;
3111                 if (dst[0] != 'h' && src[0] == 'h')
3112                         instr->type = INSTR_ALU_OR_MH;
3113                 if (dst[0] == 'h' && src[0] != 'h')
3114                         instr->type = INSTR_ALU_OR_HM;
3115                 if (dst[0] == 'h' && src[0] == 'h')
3116                         instr->type = INSTR_ALU_OR_HH;
3117
3118                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3119                 instr->alu.dst.n_bits = fdst->n_bits;
3120                 instr->alu.dst.offset = fdst->offset / 8;
3121                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3122                 instr->alu.src.n_bits = fsrc->n_bits;
3123                 instr->alu.src.offset = fsrc->offset / 8;
3124                 return 0;
3125         }
3126
3127         /* OR_I. */
3128         src_val = strtoull(src, &src, 0);
3129         CHECK(!src[0], EINVAL);
3130
3131         if (dst[0] == 'h')
3132                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3133
3134         instr->type = INSTR_ALU_OR_I;
3135         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3136         instr->alu.dst.n_bits = fdst->n_bits;
3137         instr->alu.dst.offset = fdst->offset / 8;
3138         instr->alu.src_val = src_val;
3139         return 0;
3140 }
3141
3142 static int
3143 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3144                         struct action *action,
3145                         char **tokens,
3146                         int n_tokens,
3147                         struct instruction *instr,
3148                         struct instruction_data *data __rte_unused)
3149 {
3150         char *dst = tokens[1], *src = tokens[2];
3151         struct field *fdst, *fsrc;
3152         uint64_t src_val;
3153         uint32_t dst_struct_id = 0, src_struct_id = 0;
3154
3155         CHECK(n_tokens == 3, EINVAL);
3156
3157         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3158         CHECK(fdst, EINVAL);
3159         CHECK(!fdst->var_size, EINVAL);
3160
3161         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3162         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3163         if (fsrc) {
3164                 CHECK(!fsrc->var_size, EINVAL);
3165
3166                 instr->type = INSTR_ALU_XOR;
3167                 if (dst[0] != 'h' && src[0] == 'h')
3168                         instr->type = INSTR_ALU_XOR_MH;
3169                 if (dst[0] == 'h' && src[0] != 'h')
3170                         instr->type = INSTR_ALU_XOR_HM;
3171                 if (dst[0] == 'h' && src[0] == 'h')
3172                         instr->type = INSTR_ALU_XOR_HH;
3173
3174                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3175                 instr->alu.dst.n_bits = fdst->n_bits;
3176                 instr->alu.dst.offset = fdst->offset / 8;
3177                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3178                 instr->alu.src.n_bits = fsrc->n_bits;
3179                 instr->alu.src.offset = fsrc->offset / 8;
3180                 return 0;
3181         }
3182
3183         /* XOR_I. */
3184         src_val = strtoull(src, &src, 0);
3185         CHECK(!src[0], EINVAL);
3186
3187         if (dst[0] == 'h')
3188                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3189
3190         instr->type = INSTR_ALU_XOR_I;
3191         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3192         instr->alu.dst.n_bits = fdst->n_bits;
3193         instr->alu.dst.offset = fdst->offset / 8;
3194         instr->alu.src_val = src_val;
3195         return 0;
3196 }
3197
3198 static inline void
3199 instr_alu_add_exec(struct rte_swx_pipeline *p)
3200 {
3201         struct thread *t = &p->threads[p->thread_id];
3202         struct instruction *ip = t->ip;
3203
3204         /* Structs */
3205         __instr_alu_add_exec(p, t, ip);
3206
3207         /* Thread. */
3208         thread_ip_inc(p);
3209 }
3210
3211 static inline void
3212 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3213 {
3214         struct thread *t = &p->threads[p->thread_id];
3215         struct instruction *ip = t->ip;
3216
3217         /* Structs. */
3218         __instr_alu_add_mh_exec(p, t, ip);
3219
3220         /* Thread. */
3221         thread_ip_inc(p);
3222 }
3223
3224 static inline void
3225 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3226 {
3227         struct thread *t = &p->threads[p->thread_id];
3228         struct instruction *ip = t->ip;
3229
3230         /* Structs. */
3231         __instr_alu_add_hm_exec(p, t, ip);
3232
3233         /* Thread. */
3234         thread_ip_inc(p);
3235 }
3236
3237 static inline void
3238 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3239 {
3240         struct thread *t = &p->threads[p->thread_id];
3241         struct instruction *ip = t->ip;
3242
3243         /* Structs. */
3244         __instr_alu_add_hh_exec(p, t, ip);
3245
3246         /* Thread. */
3247         thread_ip_inc(p);
3248 }
3249
3250 static inline void
3251 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3252 {
3253         struct thread *t = &p->threads[p->thread_id];
3254         struct instruction *ip = t->ip;
3255
3256         /* Structs. */
3257         __instr_alu_add_mi_exec(p, t, ip);
3258
3259         /* Thread. */
3260         thread_ip_inc(p);
3261 }
3262
3263 static inline void
3264 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3265 {
3266         struct thread *t = &p->threads[p->thread_id];
3267         struct instruction *ip = t->ip;
3268
3269         /* Structs. */
3270         __instr_alu_add_hi_exec(p, t, ip);
3271
3272         /* Thread. */
3273         thread_ip_inc(p);
3274 }
3275
3276 static inline void
3277 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3278 {
3279         struct thread *t = &p->threads[p->thread_id];
3280         struct instruction *ip = t->ip;
3281
3282         /* Structs. */
3283         __instr_alu_sub_exec(p, t, ip);
3284
3285         /* Thread. */
3286         thread_ip_inc(p);
3287 }
3288
3289 static inline void
3290 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3291 {
3292         struct thread *t = &p->threads[p->thread_id];
3293         struct instruction *ip = t->ip;
3294
3295         /* Structs. */
3296         __instr_alu_sub_mh_exec(p, t, ip);
3297
3298         /* Thread. */
3299         thread_ip_inc(p);
3300 }
3301
3302 static inline void
3303 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3304 {
3305         struct thread *t = &p->threads[p->thread_id];
3306         struct instruction *ip = t->ip;
3307
3308         /* Structs. */
3309         __instr_alu_sub_hm_exec(p, t, ip);
3310
3311         /* Thread. */
3312         thread_ip_inc(p);
3313 }
3314
3315 static inline void
3316 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3317 {
3318         struct thread *t = &p->threads[p->thread_id];
3319         struct instruction *ip = t->ip;
3320
3321         /* Structs. */
3322         __instr_alu_sub_hh_exec(p, t, ip);
3323
3324         /* Thread. */
3325         thread_ip_inc(p);
3326 }
3327
3328 static inline void
3329 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3330 {
3331         struct thread *t = &p->threads[p->thread_id];
3332         struct instruction *ip = t->ip;
3333
3334         /* Structs. */
3335         __instr_alu_sub_mi_exec(p, t, ip);
3336
3337         /* Thread. */
3338         thread_ip_inc(p);
3339 }
3340
3341 static inline void
3342 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3343 {
3344         struct thread *t = &p->threads[p->thread_id];
3345         struct instruction *ip = t->ip;
3346
3347         /* Structs. */
3348         __instr_alu_sub_hi_exec(p, t, ip);
3349
3350         /* Thread. */
3351         thread_ip_inc(p);
3352 }
3353
3354 static inline void
3355 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3356 {
3357         struct thread *t = &p->threads[p->thread_id];
3358         struct instruction *ip = t->ip;
3359
3360         /* Structs. */
3361         __instr_alu_shl_exec(p, t, ip);
3362
3363         /* Thread. */
3364         thread_ip_inc(p);
3365 }
3366
3367 static inline void
3368 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3369 {
3370         struct thread *t = &p->threads[p->thread_id];
3371         struct instruction *ip = t->ip;
3372
3373         /* Structs. */
3374         __instr_alu_shl_mh_exec(p, t, ip);
3375
3376         /* Thread. */
3377         thread_ip_inc(p);
3378 }
3379
3380 static inline void
3381 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3382 {
3383         struct thread *t = &p->threads[p->thread_id];
3384         struct instruction *ip = t->ip;
3385
3386         /* Structs. */
3387         __instr_alu_shl_hm_exec(p, t, ip);
3388
3389         /* Thread. */
3390         thread_ip_inc(p);
3391 }
3392
3393 static inline void
3394 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3395 {
3396         struct thread *t = &p->threads[p->thread_id];
3397         struct instruction *ip = t->ip;
3398
3399         /* Structs. */
3400         __instr_alu_shl_hh_exec(p, t, ip);
3401
3402         /* Thread. */
3403         thread_ip_inc(p);
3404 }
3405
3406 static inline void
3407 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3408 {
3409         struct thread *t = &p->threads[p->thread_id];
3410         struct instruction *ip = t->ip;
3411
3412         /* Structs. */
3413         __instr_alu_shl_mi_exec(p, t, ip);
3414
3415         /* Thread. */
3416         thread_ip_inc(p);
3417 }
3418
3419 static inline void
3420 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3421 {
3422         struct thread *t = &p->threads[p->thread_id];
3423         struct instruction *ip = t->ip;
3424
3425         /* Structs. */
3426         __instr_alu_shl_hi_exec(p, t, ip);
3427
3428         /* Thread. */
3429         thread_ip_inc(p);
3430 }
3431
3432 static inline void
3433 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3434 {
3435         struct thread *t = &p->threads[p->thread_id];
3436         struct instruction *ip = t->ip;
3437
3438         /* Structs. */
3439         __instr_alu_shr_exec(p, t, ip);
3440
3441         /* Thread. */
3442         thread_ip_inc(p);
3443 }
3444
3445 static inline void
3446 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3447 {
3448         struct thread *t = &p->threads[p->thread_id];
3449         struct instruction *ip = t->ip;
3450
3451         /* Structs. */
3452         __instr_alu_shr_mh_exec(p, t, ip);
3453
3454         /* Thread. */
3455         thread_ip_inc(p);
3456 }
3457
3458 static inline void
3459 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3460 {
3461         struct thread *t = &p->threads[p->thread_id];
3462         struct instruction *ip = t->ip;
3463
3464         /* Structs. */
3465         __instr_alu_shr_hm_exec(p, t, ip);
3466
3467         /* Thread. */
3468         thread_ip_inc(p);
3469 }
3470
3471 static inline void
3472 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3473 {
3474         struct thread *t = &p->threads[p->thread_id];
3475         struct instruction *ip = t->ip;
3476
3477         /* Structs. */
3478         __instr_alu_shr_hh_exec(p, t, ip);
3479
3480         /* Thread. */
3481         thread_ip_inc(p);
3482 }
3483
3484 static inline void
3485 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3486 {
3487         struct thread *t = &p->threads[p->thread_id];
3488         struct instruction *ip = t->ip;
3489
3490         /* Structs. */
3491         __instr_alu_shr_mi_exec(p, t, ip);
3492
3493         /* Thread. */
3494         thread_ip_inc(p);
3495 }
3496
3497 static inline void
3498 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3499 {
3500         struct thread *t = &p->threads[p->thread_id];
3501         struct instruction *ip = t->ip;
3502
3503         /* Structs. */
3504         __instr_alu_shr_hi_exec(p, t, ip);
3505
3506         /* Thread. */
3507         thread_ip_inc(p);
3508 }
3509
3510 static inline void
3511 instr_alu_and_exec(struct rte_swx_pipeline *p)
3512 {
3513         struct thread *t = &p->threads[p->thread_id];
3514         struct instruction *ip = t->ip;
3515
3516         /* Structs. */
3517         __instr_alu_and_exec(p, t, ip);
3518
3519         /* Thread. */
3520         thread_ip_inc(p);
3521 }
3522
3523 static inline void
3524 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3525 {
3526         struct thread *t = &p->threads[p->thread_id];
3527         struct instruction *ip = t->ip;
3528
3529         /* Structs. */
3530         __instr_alu_and_mh_exec(p, t, ip);
3531
3532         /* Thread. */
3533         thread_ip_inc(p);
3534 }
3535
3536 static inline void
3537 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3538 {
3539         struct thread *t = &p->threads[p->thread_id];
3540         struct instruction *ip = t->ip;
3541
3542         /* Structs. */
3543         __instr_alu_and_hm_exec(p, t, ip);
3544
3545         /* Thread. */
3546         thread_ip_inc(p);
3547 }
3548
3549 static inline void
3550 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3551 {
3552         struct thread *t = &p->threads[p->thread_id];
3553         struct instruction *ip = t->ip;
3554
3555         /* Structs. */
3556         __instr_alu_and_hh_exec(p, t, ip);
3557
3558         /* Thread. */
3559         thread_ip_inc(p);
3560 }
3561
3562 static inline void
3563 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3564 {
3565         struct thread *t = &p->threads[p->thread_id];
3566         struct instruction *ip = t->ip;
3567
3568         /* Structs. */
3569         __instr_alu_and_i_exec(p, t, ip);
3570
3571         /* Thread. */
3572         thread_ip_inc(p);
3573 }
3574
3575 static inline void
3576 instr_alu_or_exec(struct rte_swx_pipeline *p)
3577 {
3578         struct thread *t = &p->threads[p->thread_id];
3579         struct instruction *ip = t->ip;
3580
3581         /* Structs. */
3582         __instr_alu_or_exec(p, t, ip);
3583
3584         /* Thread. */
3585         thread_ip_inc(p);
3586 }
3587
3588 static inline void
3589 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3590 {
3591         struct thread *t = &p->threads[p->thread_id];
3592         struct instruction *ip = t->ip;
3593
3594         /* Structs. */
3595         __instr_alu_or_mh_exec(p, t, ip);
3596
3597         /* Thread. */
3598         thread_ip_inc(p);
3599 }
3600
3601 static inline void
3602 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3603 {
3604         struct thread *t = &p->threads[p->thread_id];
3605         struct instruction *ip = t->ip;
3606
3607         /* Structs. */
3608         __instr_alu_or_hm_exec(p, t, ip);
3609
3610         /* Thread. */
3611         thread_ip_inc(p);
3612 }
3613
3614 static inline void
3615 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3616 {
3617         struct thread *t = &p->threads[p->thread_id];
3618         struct instruction *ip = t->ip;
3619
3620         /* Structs. */
3621         __instr_alu_or_hh_exec(p, t, ip);
3622
3623         /* Thread. */
3624         thread_ip_inc(p);
3625 }
3626
3627 static inline void
3628 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3629 {
3630         struct thread *t = &p->threads[p->thread_id];
3631         struct instruction *ip = t->ip;
3632
3633         /* Structs. */
3634         __instr_alu_or_i_exec(p, t, ip);
3635
3636         /* Thread. */
3637         thread_ip_inc(p);
3638 }
3639
3640 static inline void
3641 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3642 {
3643         struct thread *t = &p->threads[p->thread_id];
3644         struct instruction *ip = t->ip;
3645
3646         /* Structs. */
3647         __instr_alu_xor_exec(p, t, ip);
3648
3649         /* Thread. */
3650         thread_ip_inc(p);
3651 }
3652
3653 static inline void
3654 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3655 {
3656         struct thread *t = &p->threads[p->thread_id];
3657         struct instruction *ip = t->ip;
3658
3659         /* Structs. */
3660         __instr_alu_xor_mh_exec(p, t, ip);
3661
3662         /* Thread. */
3663         thread_ip_inc(p);
3664 }
3665
3666 static inline void
3667 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3668 {
3669         struct thread *t = &p->threads[p->thread_id];
3670         struct instruction *ip = t->ip;
3671
3672         /* Structs. */
3673         __instr_alu_xor_hm_exec(p, t, ip);
3674
3675         /* Thread. */
3676         thread_ip_inc(p);
3677 }
3678
3679 static inline void
3680 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3681 {
3682         struct thread *t = &p->threads[p->thread_id];
3683         struct instruction *ip = t->ip;
3684
3685         /* Structs. */
3686         __instr_alu_xor_hh_exec(p, t, ip);
3687
3688         /* Thread. */
3689         thread_ip_inc(p);
3690 }
3691
3692 static inline void
3693 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3694 {
3695         struct thread *t = &p->threads[p->thread_id];
3696         struct instruction *ip = t->ip;
3697
3698         /* Structs. */
3699         __instr_alu_xor_i_exec(p, t, ip);
3700
3701         /* Thread. */
3702         thread_ip_inc(p);
3703 }
3704
3705 static inline void
3706 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3707 {
3708         struct thread *t = &p->threads[p->thread_id];
3709         struct instruction *ip = t->ip;
3710
3711         /* Structs. */
3712         __instr_alu_ckadd_field_exec(p, t, ip);
3713
3714         /* Thread. */
3715         thread_ip_inc(p);
3716 }
3717
3718 static inline void
3719 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3720 {
3721         struct thread *t = &p->threads[p->thread_id];
3722         struct instruction *ip = t->ip;
3723
3724         /* Structs. */
3725         __instr_alu_cksub_field_exec(p, t, ip);
3726
3727         /* Thread. */
3728         thread_ip_inc(p);
3729 }
3730
3731 static inline void
3732 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3733 {
3734         struct thread *t = &p->threads[p->thread_id];
3735         struct instruction *ip = t->ip;
3736
3737         /* Structs. */
3738         __instr_alu_ckadd_struct20_exec(p, t, ip);
3739
3740         /* Thread. */
3741         thread_ip_inc(p);
3742 }
3743
3744 static inline void
3745 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3746 {
3747         struct thread *t = &p->threads[p->thread_id];
3748         struct instruction *ip = t->ip;
3749
3750         /* Structs. */
3751         __instr_alu_ckadd_struct_exec(p, t, ip);
3752
3753         /* Thread. */
3754         thread_ip_inc(p);
3755 }
3756
3757 /*
3758  * Register array.
3759  */
3760 static struct regarray *
3761 regarray_find(struct rte_swx_pipeline *p, const char *name);
3762
3763 static int
3764 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3765                       struct action *action,
3766                       char **tokens,
3767                       int n_tokens,
3768                       struct instruction *instr,
3769                       struct instruction_data *data __rte_unused)
3770 {
3771         char *regarray = tokens[1], *idx = tokens[2];
3772         struct regarray *r;
3773         struct field *fidx;
3774         uint32_t idx_struct_id, idx_val;
3775
3776         CHECK(n_tokens == 3, EINVAL);
3777
3778         r = regarray_find(p, regarray);
3779         CHECK(r, EINVAL);
3780
3781         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3782         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3783         if (fidx) {
3784                 CHECK(!fidx->var_size, EINVAL);
3785
3786                 instr->type = INSTR_REGPREFETCH_RM;
3787                 if (idx[0] == 'h')
3788                         instr->type = INSTR_REGPREFETCH_RH;
3789
3790                 instr->regarray.regarray_id = r->id;
3791                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3792                 instr->regarray.idx.n_bits = fidx->n_bits;
3793                 instr->regarray.idx.offset = fidx->offset / 8;
3794                 instr->regarray.dstsrc_val = 0; /* Unused. */
3795                 return 0;
3796         }
3797
3798         /* REGPREFETCH_RI. */
3799         idx_val = strtoul(idx, &idx, 0);
3800         CHECK(!idx[0], EINVAL);
3801
3802         instr->type = INSTR_REGPREFETCH_RI;
3803         instr->regarray.regarray_id = r->id;
3804         instr->regarray.idx_val = idx_val;
3805         instr->regarray.dstsrc_val = 0; /* Unused. */
3806         return 0;
3807 }
3808
3809 static int
3810 instr_regrd_translate(struct rte_swx_pipeline *p,
3811                       struct action *action,
3812                       char **tokens,
3813                       int n_tokens,
3814                       struct instruction *instr,
3815                       struct instruction_data *data __rte_unused)
3816 {
3817         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3818         struct regarray *r;
3819         struct field *fdst, *fidx;
3820         uint32_t dst_struct_id, idx_struct_id, idx_val;
3821
3822         CHECK(n_tokens == 4, EINVAL);
3823
3824         r = regarray_find(p, regarray);
3825         CHECK(r, EINVAL);
3826
3827         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3828         CHECK(fdst, EINVAL);
3829         CHECK(!fdst->var_size, EINVAL);
3830
3831         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3832         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3833         if (fidx) {
3834                 CHECK(!fidx->var_size, EINVAL);
3835
3836                 instr->type = INSTR_REGRD_MRM;
3837                 if (dst[0] == 'h' && idx[0] != 'h')
3838                         instr->type = INSTR_REGRD_HRM;
3839                 if (dst[0] != 'h' && idx[0] == 'h')
3840                         instr->type = INSTR_REGRD_MRH;
3841                 if (dst[0] == 'h' && idx[0] == 'h')
3842                         instr->type = INSTR_REGRD_HRH;
3843
3844                 instr->regarray.regarray_id = r->id;
3845                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3846                 instr->regarray.idx.n_bits = fidx->n_bits;
3847                 instr->regarray.idx.offset = fidx->offset / 8;
3848                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3849                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3850                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3851                 return 0;
3852         }
3853
3854         /* REGRD_MRI, REGRD_HRI. */
3855         idx_val = strtoul(idx, &idx, 0);
3856         CHECK(!idx[0], EINVAL);
3857
3858         instr->type = INSTR_REGRD_MRI;
3859         if (dst[0] == 'h')
3860                 instr->type = INSTR_REGRD_HRI;
3861
3862         instr->regarray.regarray_id = r->id;
3863         instr->regarray.idx_val = idx_val;
3864         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3865         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3866         instr->regarray.dstsrc.offset = fdst->offset / 8;
3867         return 0;
3868 }
3869
3870 static int
3871 instr_regwr_translate(struct rte_swx_pipeline *p,
3872                       struct action *action,
3873                       char **tokens,
3874                       int n_tokens,
3875                       struct instruction *instr,
3876                       struct instruction_data *data __rte_unused)
3877 {
3878         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3879         struct regarray *r;
3880         struct field *fidx, *fsrc;
3881         uint64_t src_val;
3882         uint32_t idx_struct_id, idx_val, src_struct_id;
3883
3884         CHECK(n_tokens == 4, EINVAL);
3885
3886         r = regarray_find(p, regarray);
3887         CHECK(r, EINVAL);
3888
3889         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3890         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3891         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3892         if (fidx && fsrc) {
3893                 CHECK(!fidx->var_size, EINVAL);
3894                 CHECK(!fsrc->var_size, EINVAL);
3895
3896                 instr->type = INSTR_REGWR_RMM;
3897                 if (idx[0] == 'h' && src[0] != 'h')
3898                         instr->type = INSTR_REGWR_RHM;
3899                 if (idx[0] != 'h' && src[0] == 'h')
3900                         instr->type = INSTR_REGWR_RMH;
3901                 if (idx[0] == 'h' && src[0] == 'h')
3902                         instr->type = INSTR_REGWR_RHH;
3903
3904                 instr->regarray.regarray_id = r->id;
3905                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3906                 instr->regarray.idx.n_bits = fidx->n_bits;
3907                 instr->regarray.idx.offset = fidx->offset / 8;
3908                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3909                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3910                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3911                 return 0;
3912         }
3913
3914         /* REGWR_RHI, REGWR_RMI. */
3915         if (fidx && !fsrc) {
3916                 CHECK(!fidx->var_size, EINVAL);
3917
3918                 src_val = strtoull(src, &src, 0);
3919                 CHECK(!src[0], EINVAL);
3920
3921                 instr->type = INSTR_REGWR_RMI;
3922                 if (idx[0] == 'h')
3923                         instr->type = INSTR_REGWR_RHI;
3924
3925                 instr->regarray.regarray_id = r->id;
3926                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3927                 instr->regarray.idx.n_bits = fidx->n_bits;
3928                 instr->regarray.idx.offset = fidx->offset / 8;
3929                 instr->regarray.dstsrc_val = src_val;
3930                 return 0;
3931         }
3932
3933         /* REGWR_RIH, REGWR_RIM. */
3934         if (!fidx && fsrc) {
3935                 idx_val = strtoul(idx, &idx, 0);
3936                 CHECK(!idx[0], EINVAL);
3937
3938                 CHECK(!fsrc->var_size, EINVAL);
3939
3940                 instr->type = INSTR_REGWR_RIM;
3941                 if (src[0] == 'h')
3942                         instr->type = INSTR_REGWR_RIH;
3943
3944                 instr->regarray.regarray_id = r->id;
3945                 instr->regarray.idx_val = idx_val;
3946                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3947                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3948                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3949                 return 0;
3950         }
3951
3952         /* REGWR_RII. */
3953         src_val = strtoull(src, &src, 0);
3954         CHECK(!src[0], EINVAL);
3955
3956         idx_val = strtoul(idx, &idx, 0);
3957         CHECK(!idx[0], EINVAL);
3958
3959         instr->type = INSTR_REGWR_RII;
3960         instr->regarray.idx_val = idx_val;
3961         instr->regarray.dstsrc_val = src_val;
3962
3963         return 0;
3964 }
3965
3966 static int
3967 instr_regadd_translate(struct rte_swx_pipeline *p,
3968                        struct action *action,
3969                        char **tokens,
3970                        int n_tokens,
3971                        struct instruction *instr,
3972                        struct instruction_data *data __rte_unused)
3973 {
3974         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3975         struct regarray *r;
3976         struct field *fidx, *fsrc;
3977         uint64_t src_val;
3978         uint32_t idx_struct_id, idx_val, src_struct_id;
3979
3980         CHECK(n_tokens == 4, EINVAL);
3981
3982         r = regarray_find(p, regarray);
3983         CHECK(r, EINVAL);
3984
3985         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
3986         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3987         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3988         if (fidx && fsrc) {
3989                 CHECK(!fidx->var_size, EINVAL);
3990                 CHECK(!fsrc->var_size, EINVAL);
3991
3992                 instr->type = INSTR_REGADD_RMM;
3993                 if (idx[0] == 'h' && src[0] != 'h')
3994                         instr->type = INSTR_REGADD_RHM;
3995                 if (idx[0] != 'h' && src[0] == 'h')
3996                         instr->type = INSTR_REGADD_RMH;
3997                 if (idx[0] == 'h' && src[0] == 'h')
3998                         instr->type = INSTR_REGADD_RHH;
3999
4000                 instr->regarray.regarray_id = r->id;
4001                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4002                 instr->regarray.idx.n_bits = fidx->n_bits;
4003                 instr->regarray.idx.offset = fidx->offset / 8;
4004                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4005                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4006                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4007                 return 0;
4008         }
4009
4010         /* REGADD_RHI, REGADD_RMI. */
4011         if (fidx && !fsrc) {
4012                 CHECK(!fidx->var_size, EINVAL);
4013
4014                 src_val = strtoull(src, &src, 0);
4015                 CHECK(!src[0], EINVAL);
4016
4017                 instr->type = INSTR_REGADD_RMI;
4018                 if (idx[0] == 'h')
4019                         instr->type = INSTR_REGADD_RHI;
4020
4021                 instr->regarray.regarray_id = r->id;
4022                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4023                 instr->regarray.idx.n_bits = fidx->n_bits;
4024                 instr->regarray.idx.offset = fidx->offset / 8;
4025                 instr->regarray.dstsrc_val = src_val;
4026                 return 0;
4027         }
4028
4029         /* REGADD_RIH, REGADD_RIM. */
4030         if (!fidx && fsrc) {
4031                 idx_val = strtoul(idx, &idx, 0);
4032                 CHECK(!idx[0], EINVAL);
4033
4034                 CHECK(!fsrc->var_size, EINVAL);
4035
4036                 instr->type = INSTR_REGADD_RIM;
4037                 if (src[0] == 'h')
4038                         instr->type = INSTR_REGADD_RIH;
4039
4040                 instr->regarray.regarray_id = r->id;
4041                 instr->regarray.idx_val = idx_val;
4042                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4043                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4044                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4045                 return 0;
4046         }
4047
4048         /* REGADD_RII. */
4049         src_val = strtoull(src, &src, 0);
4050         CHECK(!src[0], EINVAL);
4051
4052         idx_val = strtoul(idx, &idx, 0);
4053         CHECK(!idx[0], EINVAL);
4054
4055         instr->type = INSTR_REGADD_RII;
4056         instr->regarray.idx_val = idx_val;
4057         instr->regarray.dstsrc_val = src_val;
4058         return 0;
4059 }
4060
4061 static inline void
4062 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4063 {
4064         struct thread *t = &p->threads[p->thread_id];
4065         struct instruction *ip = t->ip;
4066
4067         /* Structs. */
4068         __instr_regprefetch_rh_exec(p, t, ip);
4069
4070         /* Thread. */
4071         thread_ip_inc(p);
4072 }
4073
4074 static inline void
4075 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4076 {
4077         struct thread *t = &p->threads[p->thread_id];
4078         struct instruction *ip = t->ip;
4079
4080         /* Structs. */
4081         __instr_regprefetch_rm_exec(p, t, ip);
4082
4083         /* Thread. */
4084         thread_ip_inc(p);
4085 }
4086
4087 static inline void
4088 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4089 {
4090         struct thread *t = &p->threads[p->thread_id];
4091         struct instruction *ip = t->ip;
4092
4093         /* Structs. */
4094         __instr_regprefetch_ri_exec(p, t, ip);
4095
4096         /* Thread. */
4097         thread_ip_inc(p);
4098 }
4099
4100 static inline void
4101 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4102 {
4103         struct thread *t = &p->threads[p->thread_id];
4104         struct instruction *ip = t->ip;
4105
4106         /* Structs. */
4107         __instr_regrd_hrh_exec(p, t, ip);
4108
4109         /* Thread. */
4110         thread_ip_inc(p);
4111 }
4112
4113 static inline void
4114 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4115 {
4116         struct thread *t = &p->threads[p->thread_id];
4117         struct instruction *ip = t->ip;
4118
4119         /* Structs. */
4120         __instr_regrd_hrm_exec(p, t, ip);
4121
4122         /* Thread. */
4123         thread_ip_inc(p);
4124 }
4125
4126 static inline void
4127 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4128 {
4129         struct thread *t = &p->threads[p->thread_id];
4130         struct instruction *ip = t->ip;
4131
4132         /* Structs. */
4133         __instr_regrd_mrh_exec(p, t, ip);
4134
4135         /* Thread. */
4136         thread_ip_inc(p);
4137 }
4138
4139 static inline void
4140 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4141 {
4142         struct thread *t = &p->threads[p->thread_id];
4143         struct instruction *ip = t->ip;
4144
4145         /* Structs. */
4146         __instr_regrd_mrm_exec(p, t, ip);
4147
4148         /* Thread. */
4149         thread_ip_inc(p);
4150 }
4151
4152 static inline void
4153 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4154 {
4155         struct thread *t = &p->threads[p->thread_id];
4156         struct instruction *ip = t->ip;
4157
4158         /* Structs. */
4159         __instr_regrd_hri_exec(p, t, ip);
4160
4161         /* Thread. */
4162         thread_ip_inc(p);
4163 }
4164
4165 static inline void
4166 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4167 {
4168         struct thread *t = &p->threads[p->thread_id];
4169         struct instruction *ip = t->ip;
4170
4171         /* Structs. */
4172         __instr_regrd_mri_exec(p, t, ip);
4173
4174         /* Thread. */
4175         thread_ip_inc(p);
4176 }
4177
4178 static inline void
4179 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4180 {
4181         struct thread *t = &p->threads[p->thread_id];
4182         struct instruction *ip = t->ip;
4183
4184         /* Structs. */
4185         __instr_regwr_rhh_exec(p, t, ip);
4186
4187         /* Thread. */
4188         thread_ip_inc(p);
4189 }
4190
4191 static inline void
4192 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4193 {
4194         struct thread *t = &p->threads[p->thread_id];
4195         struct instruction *ip = t->ip;
4196
4197         /* Structs. */
4198         __instr_regwr_rhm_exec(p, t, ip);
4199
4200         /* Thread. */
4201         thread_ip_inc(p);
4202 }
4203
4204 static inline void
4205 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4206 {
4207         struct thread *t = &p->threads[p->thread_id];
4208         struct instruction *ip = t->ip;
4209
4210         /* Structs. */
4211         __instr_regwr_rmh_exec(p, t, ip);
4212
4213         /* Thread. */
4214         thread_ip_inc(p);
4215 }
4216
4217 static inline void
4218 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4219 {
4220         struct thread *t = &p->threads[p->thread_id];
4221         struct instruction *ip = t->ip;
4222
4223         /* Structs. */
4224         __instr_regwr_rmm_exec(p, t, ip);
4225
4226         /* Thread. */
4227         thread_ip_inc(p);
4228 }
4229
4230 static inline void
4231 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4232 {
4233         struct thread *t = &p->threads[p->thread_id];
4234         struct instruction *ip = t->ip;
4235
4236         /* Structs. */
4237         __instr_regwr_rhi_exec(p, t, ip);
4238
4239         /* Thread. */
4240         thread_ip_inc(p);
4241 }
4242
4243 static inline void
4244 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4245 {
4246         struct thread *t = &p->threads[p->thread_id];
4247         struct instruction *ip = t->ip;
4248
4249         /* Structs. */
4250         __instr_regwr_rmi_exec(p, t, ip);
4251
4252         /* Thread. */
4253         thread_ip_inc(p);
4254 }
4255
4256 static inline void
4257 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4258 {
4259         struct thread *t = &p->threads[p->thread_id];
4260         struct instruction *ip = t->ip;
4261
4262         /* Structs. */
4263         __instr_regwr_rih_exec(p, t, ip);
4264
4265         /* Thread. */
4266         thread_ip_inc(p);
4267 }
4268
4269 static inline void
4270 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4271 {
4272         struct thread *t = &p->threads[p->thread_id];
4273         struct instruction *ip = t->ip;
4274
4275         /* Structs. */
4276         __instr_regwr_rim_exec(p, t, ip);
4277
4278         /* Thread. */
4279         thread_ip_inc(p);
4280 }
4281
4282 static inline void
4283 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4284 {
4285         struct thread *t = &p->threads[p->thread_id];
4286         struct instruction *ip = t->ip;
4287
4288         /* Structs. */
4289         __instr_regwr_rii_exec(p, t, ip);
4290
4291         /* Thread. */
4292         thread_ip_inc(p);
4293 }
4294
4295 static inline void
4296 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4297 {
4298         struct thread *t = &p->threads[p->thread_id];
4299         struct instruction *ip = t->ip;
4300
4301         /* Structs. */
4302         __instr_regadd_rhh_exec(p, t, ip);
4303
4304         /* Thread. */
4305         thread_ip_inc(p);
4306 }
4307
4308 static inline void
4309 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4310 {
4311         struct thread *t = &p->threads[p->thread_id];
4312         struct instruction *ip = t->ip;
4313
4314         /* Structs. */
4315         __instr_regadd_rhm_exec(p, t, ip);
4316
4317         /* Thread. */
4318         thread_ip_inc(p);
4319 }
4320
4321 static inline void
4322 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4323 {
4324         struct thread *t = &p->threads[p->thread_id];
4325         struct instruction *ip = t->ip;
4326
4327         /* Structs. */
4328         __instr_regadd_rmh_exec(p, t, ip);
4329
4330         /* Thread. */
4331         thread_ip_inc(p);
4332 }
4333
4334 static inline void
4335 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4336 {
4337         struct thread *t = &p->threads[p->thread_id];
4338         struct instruction *ip = t->ip;
4339
4340         /* Structs. */
4341         __instr_regadd_rmm_exec(p, t, ip);
4342
4343         /* Thread. */
4344         thread_ip_inc(p);
4345 }
4346
4347 static inline void
4348 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4349 {
4350         struct thread *t = &p->threads[p->thread_id];
4351         struct instruction *ip = t->ip;
4352
4353         /* Structs. */
4354         __instr_regadd_rhi_exec(p, t, ip);
4355
4356         /* Thread. */
4357         thread_ip_inc(p);
4358 }
4359
4360 static inline void
4361 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4362 {
4363         struct thread *t = &p->threads[p->thread_id];
4364         struct instruction *ip = t->ip;
4365
4366         /* Structs. */
4367         __instr_regadd_rmi_exec(p, t, ip);
4368
4369         /* Thread. */
4370         thread_ip_inc(p);
4371 }
4372
4373 static inline void
4374 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4375 {
4376         struct thread *t = &p->threads[p->thread_id];
4377         struct instruction *ip = t->ip;
4378
4379         /* Structs. */
4380         __instr_regadd_rih_exec(p, t, ip);
4381
4382         /* Thread. */
4383         thread_ip_inc(p);
4384 }
4385
4386 static inline void
4387 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4388 {
4389         struct thread *t = &p->threads[p->thread_id];
4390         struct instruction *ip = t->ip;
4391
4392         /* Structs. */
4393         __instr_regadd_rim_exec(p, t, ip);
4394
4395         /* Thread. */
4396         thread_ip_inc(p);
4397 }
4398
4399 static inline void
4400 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4401 {
4402         struct thread *t = &p->threads[p->thread_id];
4403         struct instruction *ip = t->ip;
4404
4405         /* Structs. */
4406         __instr_regadd_rii_exec(p, t, ip);
4407
4408         /* Thread. */
4409         thread_ip_inc(p);
4410 }
4411
4412 /*
4413  * metarray.
4414  */
4415 static struct metarray *
4416 metarray_find(struct rte_swx_pipeline *p, const char *name);
4417
4418 static int
4419 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4420                             struct action *action,
4421                             char **tokens,
4422                             int n_tokens,
4423                             struct instruction *instr,
4424                             struct instruction_data *data __rte_unused)
4425 {
4426         char *metarray = tokens[1], *idx = tokens[2];
4427         struct metarray *m;
4428         struct field *fidx;
4429         uint32_t idx_struct_id, idx_val;
4430
4431         CHECK(n_tokens == 3, EINVAL);
4432
4433         m = metarray_find(p, metarray);
4434         CHECK(m, EINVAL);
4435
4436         /* METPREFETCH_H, METPREFETCH_M. */
4437         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4438         if (fidx) {
4439                 CHECK(!fidx->var_size, EINVAL);
4440
4441                 instr->type = INSTR_METPREFETCH_M;
4442                 if (idx[0] == 'h')
4443                         instr->type = INSTR_METPREFETCH_H;
4444
4445                 instr->meter.metarray_id = m->id;
4446                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4447                 instr->meter.idx.n_bits = fidx->n_bits;
4448                 instr->meter.idx.offset = fidx->offset / 8;
4449                 return 0;
4450         }
4451
4452         /* METPREFETCH_I. */
4453         idx_val = strtoul(idx, &idx, 0);
4454         CHECK(!idx[0], EINVAL);
4455
4456         instr->type = INSTR_METPREFETCH_I;
4457         instr->meter.metarray_id = m->id;
4458         instr->meter.idx_val = idx_val;
4459         return 0;
4460 }
4461
4462 static int
4463 instr_meter_translate(struct rte_swx_pipeline *p,
4464                       struct action *action,
4465                       char **tokens,
4466                       int n_tokens,
4467                       struct instruction *instr,
4468                       struct instruction_data *data __rte_unused)
4469 {
4470         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4471         char *color_in = tokens[4], *color_out = tokens[5];
4472         struct metarray *m;
4473         struct field *fidx, *flength, *fcin, *fcout;
4474         uint32_t idx_struct_id, length_struct_id;
4475         uint32_t color_in_struct_id, color_out_struct_id;
4476
4477         CHECK(n_tokens == 6, EINVAL);
4478
4479         m = metarray_find(p, metarray);
4480         CHECK(m, EINVAL);
4481
4482         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4483
4484         flength = struct_field_parse(p, action, length, &length_struct_id);
4485         CHECK(flength, EINVAL);
4486         CHECK(!flength->var_size, EINVAL);
4487
4488         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4489
4490         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4491         CHECK(fcout, EINVAL);
4492         CHECK(!fcout->var_size, EINVAL);
4493
4494         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4495         if (fidx && fcin) {
4496                 CHECK(!fidx->var_size, EINVAL);
4497                 CHECK(!fcin->var_size, EINVAL);
4498
4499                 instr->type = INSTR_METER_MMM;
4500                 if (idx[0] == 'h' && length[0] == 'h')
4501                         instr->type = INSTR_METER_HHM;
4502                 if (idx[0] == 'h' && length[0] != 'h')
4503                         instr->type = INSTR_METER_HMM;
4504                 if (idx[0] != 'h' && length[0] == 'h')
4505                         instr->type = INSTR_METER_MHM;
4506
4507                 instr->meter.metarray_id = m->id;
4508
4509                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4510                 instr->meter.idx.n_bits = fidx->n_bits;
4511                 instr->meter.idx.offset = fidx->offset / 8;
4512
4513                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4514                 instr->meter.length.n_bits = flength->n_bits;
4515                 instr->meter.length.offset = flength->offset / 8;
4516
4517                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4518                 instr->meter.color_in.n_bits = fcin->n_bits;
4519                 instr->meter.color_in.offset = fcin->offset / 8;
4520
4521                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4522                 instr->meter.color_out.n_bits = fcout->n_bits;
4523                 instr->meter.color_out.offset = fcout->offset / 8;
4524
4525                 return 0;
4526         }
4527
4528         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4529         if (fidx && !fcin) {
4530                 uint32_t color_in_val;
4531
4532                 CHECK(!fidx->var_size, EINVAL);
4533
4534                 color_in_val = strtoul(color_in, &color_in, 0);
4535                 CHECK(!color_in[0], EINVAL);
4536
4537                 instr->type = INSTR_METER_MMI;
4538                 if (idx[0] == 'h' && length[0] == 'h')
4539                         instr->type = INSTR_METER_HHI;
4540                 if (idx[0] == 'h' && length[0] != 'h')
4541                         instr->type = INSTR_METER_HMI;
4542                 if (idx[0] != 'h' && length[0] == 'h')
4543                         instr->type = INSTR_METER_MHI;
4544
4545                 instr->meter.metarray_id = m->id;
4546
4547                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4548                 instr->meter.idx.n_bits = fidx->n_bits;
4549                 instr->meter.idx.offset = fidx->offset / 8;
4550
4551                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4552                 instr->meter.length.n_bits = flength->n_bits;
4553                 instr->meter.length.offset = flength->offset / 8;
4554
4555                 instr->meter.color_in_val = color_in_val;
4556
4557                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4558                 instr->meter.color_out.n_bits = fcout->n_bits;
4559                 instr->meter.color_out.offset = fcout->offset / 8;
4560
4561                 return 0;
4562         }
4563
4564         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4565         if (!fidx && fcin) {
4566                 uint32_t idx_val;
4567
4568                 idx_val = strtoul(idx, &idx, 0);
4569                 CHECK(!idx[0], EINVAL);
4570
4571                 CHECK(!fcin->var_size, EINVAL);
4572
4573                 instr->type = INSTR_METER_IMM;
4574                 if (length[0] == 'h')
4575                         instr->type = INSTR_METER_IHM;
4576
4577                 instr->meter.metarray_id = m->id;
4578
4579                 instr->meter.idx_val = idx_val;
4580
4581                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4582                 instr->meter.length.n_bits = flength->n_bits;
4583                 instr->meter.length.offset = flength->offset / 8;
4584
4585                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4586                 instr->meter.color_in.n_bits = fcin->n_bits;
4587                 instr->meter.color_in.offset = fcin->offset / 8;
4588
4589                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4590                 instr->meter.color_out.n_bits = fcout->n_bits;
4591                 instr->meter.color_out.offset = fcout->offset / 8;
4592
4593                 return 0;
4594         }
4595
4596         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4597         if (!fidx && !fcin) {
4598                 uint32_t idx_val, color_in_val;
4599
4600                 idx_val = strtoul(idx, &idx, 0);
4601                 CHECK(!idx[0], EINVAL);
4602
4603                 color_in_val = strtoul(color_in, &color_in, 0);
4604                 CHECK(!color_in[0], EINVAL);
4605
4606                 instr->type = INSTR_METER_IMI;
4607                 if (length[0] == 'h')
4608                         instr->type = INSTR_METER_IHI;
4609
4610                 instr->meter.metarray_id = m->id;
4611
4612                 instr->meter.idx_val = idx_val;
4613
4614                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4615                 instr->meter.length.n_bits = flength->n_bits;
4616                 instr->meter.length.offset = flength->offset / 8;
4617
4618                 instr->meter.color_in_val = color_in_val;
4619
4620                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4621                 instr->meter.color_out.n_bits = fcout->n_bits;
4622                 instr->meter.color_out.offset = fcout->offset / 8;
4623
4624                 return 0;
4625         }
4626
4627         CHECK(0, EINVAL);
4628 }
4629
4630 static inline void
4631 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4632 {
4633         struct thread *t = &p->threads[p->thread_id];
4634         struct instruction *ip = t->ip;
4635
4636         /* Structs. */
4637         __instr_metprefetch_h_exec(p, t, ip);
4638
4639         /* Thread. */
4640         thread_ip_inc(p);
4641 }
4642
4643 static inline void
4644 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4645 {
4646         struct thread *t = &p->threads[p->thread_id];
4647         struct instruction *ip = t->ip;
4648
4649         /* Structs. */
4650         __instr_metprefetch_m_exec(p, t, ip);
4651
4652         /* Thread. */
4653         thread_ip_inc(p);
4654 }
4655
4656 static inline void
4657 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4658 {
4659         struct thread *t = &p->threads[p->thread_id];
4660         struct instruction *ip = t->ip;
4661
4662         /* Structs. */
4663         __instr_metprefetch_i_exec(p, t, ip);
4664
4665         /* Thread. */
4666         thread_ip_inc(p);
4667 }
4668
4669 static inline void
4670 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4671 {
4672         struct thread *t = &p->threads[p->thread_id];
4673         struct instruction *ip = t->ip;
4674
4675         /* Structs. */
4676         __instr_meter_hhm_exec(p, t, ip);
4677
4678         /* Thread. */
4679         thread_ip_inc(p);
4680 }
4681
4682 static inline void
4683 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4684 {
4685         struct thread *t = &p->threads[p->thread_id];
4686         struct instruction *ip = t->ip;
4687
4688         /* Structs. */
4689         __instr_meter_hhi_exec(p, t, ip);
4690
4691         /* Thread. */
4692         thread_ip_inc(p);
4693 }
4694
4695 static inline void
4696 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4697 {
4698         struct thread *t = &p->threads[p->thread_id];
4699         struct instruction *ip = t->ip;
4700
4701         /* Structs. */
4702         __instr_meter_hmm_exec(p, t, ip);
4703
4704         /* Thread. */
4705         thread_ip_inc(p);
4706 }
4707
4708 static inline void
4709 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4710 {
4711         struct thread *t = &p->threads[p->thread_id];
4712         struct instruction *ip = t->ip;
4713
4714         /* Structs. */
4715         __instr_meter_hmi_exec(p, t, ip);
4716
4717         /* Thread. */
4718         thread_ip_inc(p);
4719 }
4720
4721 static inline void
4722 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4723 {
4724         struct thread *t = &p->threads[p->thread_id];
4725         struct instruction *ip = t->ip;
4726
4727         /* Structs. */
4728         __instr_meter_mhm_exec(p, t, ip);
4729
4730         /* Thread. */
4731         thread_ip_inc(p);
4732 }
4733
4734 static inline void
4735 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4736 {
4737         struct thread *t = &p->threads[p->thread_id];
4738         struct instruction *ip = t->ip;
4739
4740         /* Structs. */
4741         __instr_meter_mhi_exec(p, t, ip);
4742
4743         /* Thread. */
4744         thread_ip_inc(p);
4745 }
4746
4747 static inline void
4748 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4749 {
4750         struct thread *t = &p->threads[p->thread_id];
4751         struct instruction *ip = t->ip;
4752
4753         /* Structs. */
4754         __instr_meter_mmm_exec(p, t, ip);
4755
4756         /* Thread. */
4757         thread_ip_inc(p);
4758 }
4759
4760 static inline void
4761 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4762 {
4763         struct thread *t = &p->threads[p->thread_id];
4764         struct instruction *ip = t->ip;
4765
4766         /* Structs. */
4767         __instr_meter_mmi_exec(p, t, ip);
4768
4769         /* Thread. */
4770         thread_ip_inc(p);
4771 }
4772
4773 static inline void
4774 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4775 {
4776         struct thread *t = &p->threads[p->thread_id];
4777         struct instruction *ip = t->ip;
4778
4779         /* Structs. */
4780         __instr_meter_ihm_exec(p, t, ip);
4781
4782         /* Thread. */
4783         thread_ip_inc(p);
4784 }
4785
4786 static inline void
4787 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4788 {
4789         struct thread *t = &p->threads[p->thread_id];
4790         struct instruction *ip = t->ip;
4791
4792         /* Structs. */
4793         __instr_meter_ihi_exec(p, t, ip);
4794
4795         /* Thread. */
4796         thread_ip_inc(p);
4797 }
4798
4799 static inline void
4800 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4801 {
4802         struct thread *t = &p->threads[p->thread_id];
4803         struct instruction *ip = t->ip;
4804
4805         /* Structs. */
4806         __instr_meter_imm_exec(p, t, ip);
4807
4808         /* Thread. */
4809         thread_ip_inc(p);
4810 }
4811
4812 static inline void
4813 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4814 {
4815         struct thread *t = &p->threads[p->thread_id];
4816         struct instruction *ip = t->ip;
4817
4818         /* Structs. */
4819         __instr_meter_imi_exec(p, t, ip);
4820
4821         /* Thread. */
4822         thread_ip_inc(p);
4823 }
4824
4825 /*
4826  * jmp.
4827  */
4828 static int
4829 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4830                     struct action *action __rte_unused,
4831                     char **tokens,
4832                     int n_tokens,
4833                     struct instruction *instr,
4834                     struct instruction_data *data)
4835 {
4836         CHECK(n_tokens == 2, EINVAL);
4837
4838         strcpy(data->jmp_label, tokens[1]);
4839
4840         instr->type = INSTR_JMP;
4841         instr->jmp.ip = NULL; /* Resolved later. */
4842         return 0;
4843 }
4844
4845 static int
4846 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4847                           struct action *action __rte_unused,
4848                           char **tokens,
4849                           int n_tokens,
4850                           struct instruction *instr,
4851                           struct instruction_data *data)
4852 {
4853         struct header *h;
4854
4855         CHECK(n_tokens == 3, EINVAL);
4856
4857         strcpy(data->jmp_label, tokens[1]);
4858
4859         h = header_parse(p, tokens[2]);
4860         CHECK(h, EINVAL);
4861
4862         instr->type = INSTR_JMP_VALID;
4863         instr->jmp.ip = NULL; /* Resolved later. */
4864         instr->jmp.header_id = h->id;
4865         return 0;
4866 }
4867
4868 static int
4869 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4870                             struct action *action __rte_unused,
4871                             char **tokens,
4872                             int n_tokens,
4873                             struct instruction *instr,
4874                             struct instruction_data *data)
4875 {
4876         struct header *h;
4877
4878         CHECK(n_tokens == 3, EINVAL);
4879
4880         strcpy(data->jmp_label, tokens[1]);
4881
4882         h = header_parse(p, tokens[2]);
4883         CHECK(h, EINVAL);
4884
4885         instr->type = INSTR_JMP_INVALID;
4886         instr->jmp.ip = NULL; /* Resolved later. */
4887         instr->jmp.header_id = h->id;
4888         return 0;
4889 }
4890
4891 static int
4892 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4893                         struct action *action,
4894                         char **tokens,
4895                         int n_tokens,
4896                         struct instruction *instr,
4897                         struct instruction_data *data)
4898 {
4899         CHECK(!action, EINVAL);
4900         CHECK(n_tokens == 2, EINVAL);
4901
4902         strcpy(data->jmp_label, tokens[1]);
4903
4904         instr->type = INSTR_JMP_HIT;
4905         instr->jmp.ip = NULL; /* Resolved later. */
4906         return 0;
4907 }
4908
4909 static int
4910 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4911                          struct action *action,
4912                          char **tokens,
4913                          int n_tokens,
4914                          struct instruction *instr,
4915                          struct instruction_data *data)
4916 {
4917         CHECK(!action, EINVAL);
4918         CHECK(n_tokens == 2, EINVAL);
4919
4920         strcpy(data->jmp_label, tokens[1]);
4921
4922         instr->type = INSTR_JMP_MISS;
4923         instr->jmp.ip = NULL; /* Resolved later. */
4924         return 0;
4925 }
4926
4927 static int
4928 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4929                                struct action *action,
4930                                char **tokens,
4931                                int n_tokens,
4932                                struct instruction *instr,
4933                                struct instruction_data *data)
4934 {
4935         struct action *a;
4936
4937         CHECK(!action, EINVAL);
4938         CHECK(n_tokens == 3, EINVAL);
4939
4940         strcpy(data->jmp_label, tokens[1]);
4941
4942         a = action_find(p, tokens[2]);
4943         CHECK(a, EINVAL);
4944
4945         instr->type = INSTR_JMP_ACTION_HIT;
4946         instr->jmp.ip = NULL; /* Resolved later. */
4947         instr->jmp.action_id = a->id;
4948         return 0;
4949 }
4950
4951 static int
4952 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4953                                 struct action *action,
4954                                 char **tokens,
4955                                 int n_tokens,
4956                                 struct instruction *instr,
4957                                 struct instruction_data *data)
4958 {
4959         struct action *a;
4960
4961         CHECK(!action, EINVAL);
4962         CHECK(n_tokens == 3, EINVAL);
4963
4964         strcpy(data->jmp_label, tokens[1]);
4965
4966         a = action_find(p, tokens[2]);
4967         CHECK(a, EINVAL);
4968
4969         instr->type = INSTR_JMP_ACTION_MISS;
4970         instr->jmp.ip = NULL; /* Resolved later. */
4971         instr->jmp.action_id = a->id;
4972         return 0;
4973 }
4974
4975 static int
4976 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4977                        struct action *action,
4978                        char **tokens,
4979                        int n_tokens,
4980                        struct instruction *instr,
4981                        struct instruction_data *data)
4982 {
4983         char *a = tokens[2], *b = tokens[3];
4984         struct field *fa, *fb;
4985         uint64_t b_val;
4986         uint32_t a_struct_id, b_struct_id;
4987
4988         CHECK(n_tokens == 4, EINVAL);
4989
4990         strcpy(data->jmp_label, tokens[1]);
4991
4992         fa = struct_field_parse(p, action, a, &a_struct_id);
4993         CHECK(fa, EINVAL);
4994         CHECK(!fa->var_size, EINVAL);
4995
4996         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
4997         fb = struct_field_parse(p, action, b, &b_struct_id);
4998         if (fb) {
4999                 CHECK(!fb->var_size, EINVAL);
5000
5001                 instr->type = INSTR_JMP_EQ;
5002                 if (a[0] != 'h' && b[0] == 'h')
5003                         instr->type = INSTR_JMP_EQ_MH;
5004                 if (a[0] == 'h' && b[0] != 'h')
5005                         instr->type = INSTR_JMP_EQ_HM;
5006                 if (a[0] == 'h' && b[0] == 'h')
5007                         instr->type = INSTR_JMP_EQ_HH;
5008                 instr->jmp.ip = NULL; /* Resolved later. */
5009
5010                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5011                 instr->jmp.a.n_bits = fa->n_bits;
5012                 instr->jmp.a.offset = fa->offset / 8;
5013                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5014                 instr->jmp.b.n_bits = fb->n_bits;
5015                 instr->jmp.b.offset = fb->offset / 8;
5016                 return 0;
5017         }
5018
5019         /* JMP_EQ_I. */
5020         b_val = strtoull(b, &b, 0);
5021         CHECK(!b[0], EINVAL);
5022
5023         if (a[0] == 'h')
5024                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5025
5026         instr->type = INSTR_JMP_EQ_I;
5027         instr->jmp.ip = NULL; /* Resolved later. */
5028         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5029         instr->jmp.a.n_bits = fa->n_bits;
5030         instr->jmp.a.offset = fa->offset / 8;
5031         instr->jmp.b_val = b_val;
5032         return 0;
5033 }
5034
5035 static int
5036 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5037                         struct action *action,
5038                         char **tokens,
5039                         int n_tokens,
5040                         struct instruction *instr,
5041                         struct instruction_data *data)
5042 {
5043         char *a = tokens[2], *b = tokens[3];
5044         struct field *fa, *fb;
5045         uint64_t b_val;
5046         uint32_t a_struct_id, b_struct_id;
5047
5048         CHECK(n_tokens == 4, EINVAL);
5049
5050         strcpy(data->jmp_label, tokens[1]);
5051
5052         fa = struct_field_parse(p, action, a, &a_struct_id);
5053         CHECK(fa, EINVAL);
5054         CHECK(!fa->var_size, EINVAL);
5055
5056         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5057         fb = struct_field_parse(p, action, b, &b_struct_id);
5058         if (fb) {
5059                 CHECK(!fb->var_size, EINVAL);
5060
5061                 instr->type = INSTR_JMP_NEQ;
5062                 if (a[0] != 'h' && b[0] == 'h')
5063                         instr->type = INSTR_JMP_NEQ_MH;
5064                 if (a[0] == 'h' && b[0] != 'h')
5065                         instr->type = INSTR_JMP_NEQ_HM;
5066                 if (a[0] == 'h' && b[0] == 'h')
5067                         instr->type = INSTR_JMP_NEQ_HH;
5068                 instr->jmp.ip = NULL; /* Resolved later. */
5069
5070                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5071                 instr->jmp.a.n_bits = fa->n_bits;
5072                 instr->jmp.a.offset = fa->offset / 8;
5073                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5074                 instr->jmp.b.n_bits = fb->n_bits;
5075                 instr->jmp.b.offset = fb->offset / 8;
5076                 return 0;
5077         }
5078
5079         /* JMP_NEQ_I. */
5080         b_val = strtoull(b, &b, 0);
5081         CHECK(!b[0], EINVAL);
5082
5083         if (a[0] == 'h')
5084                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5085
5086         instr->type = INSTR_JMP_NEQ_I;
5087         instr->jmp.ip = NULL; /* Resolved later. */
5088         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5089         instr->jmp.a.n_bits = fa->n_bits;
5090         instr->jmp.a.offset = fa->offset / 8;
5091         instr->jmp.b_val = b_val;
5092         return 0;
5093 }
5094
5095 static int
5096 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5097                        struct action *action,
5098                        char **tokens,
5099                        int n_tokens,
5100                        struct instruction *instr,
5101                        struct instruction_data *data)
5102 {
5103         char *a = tokens[2], *b = tokens[3];
5104         struct field *fa, *fb;
5105         uint64_t b_val;
5106         uint32_t a_struct_id, b_struct_id;
5107
5108         CHECK(n_tokens == 4, EINVAL);
5109
5110         strcpy(data->jmp_label, tokens[1]);
5111
5112         fa = struct_field_parse(p, action, a, &a_struct_id);
5113         CHECK(fa, EINVAL);
5114         CHECK(!fa->var_size, EINVAL);
5115
5116         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5117         fb = struct_field_parse(p, action, b, &b_struct_id);
5118         if (fb) {
5119                 CHECK(!fb->var_size, EINVAL);
5120
5121                 instr->type = INSTR_JMP_LT;
5122                 if (a[0] == 'h' && b[0] != 'h')
5123                         instr->type = INSTR_JMP_LT_HM;
5124                 if (a[0] != 'h' && b[0] == 'h')
5125                         instr->type = INSTR_JMP_LT_MH;
5126                 if (a[0] == 'h' && b[0] == 'h')
5127                         instr->type = INSTR_JMP_LT_HH;
5128                 instr->jmp.ip = NULL; /* Resolved later. */
5129
5130                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5131                 instr->jmp.a.n_bits = fa->n_bits;
5132                 instr->jmp.a.offset = fa->offset / 8;
5133                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5134                 instr->jmp.b.n_bits = fb->n_bits;
5135                 instr->jmp.b.offset = fb->offset / 8;
5136                 return 0;
5137         }
5138
5139         /* JMP_LT_MI, JMP_LT_HI. */
5140         b_val = strtoull(b, &b, 0);
5141         CHECK(!b[0], EINVAL);
5142
5143         instr->type = INSTR_JMP_LT_MI;
5144         if (a[0] == 'h')
5145                 instr->type = INSTR_JMP_LT_HI;
5146         instr->jmp.ip = NULL; /* Resolved later. */
5147
5148         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5149         instr->jmp.a.n_bits = fa->n_bits;
5150         instr->jmp.a.offset = fa->offset / 8;
5151         instr->jmp.b_val = b_val;
5152         return 0;
5153 }
5154
5155 static int
5156 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5157                        struct action *action,
5158                        char **tokens,
5159                        int n_tokens,
5160                        struct instruction *instr,
5161                        struct instruction_data *data)
5162 {
5163         char *a = tokens[2], *b = tokens[3];
5164         struct field *fa, *fb;
5165         uint64_t b_val;
5166         uint32_t a_struct_id, b_struct_id;
5167
5168         CHECK(n_tokens == 4, EINVAL);
5169
5170         strcpy(data->jmp_label, tokens[1]);
5171
5172         fa = struct_field_parse(p, action, a, &a_struct_id);
5173         CHECK(fa, EINVAL);
5174         CHECK(!fa->var_size, EINVAL);
5175
5176         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5177         fb = struct_field_parse(p, action, b, &b_struct_id);
5178         if (fb) {
5179                 CHECK(!fb->var_size, EINVAL);
5180
5181                 instr->type = INSTR_JMP_GT;
5182                 if (a[0] == 'h' && b[0] != 'h')
5183                         instr->type = INSTR_JMP_GT_HM;
5184                 if (a[0] != 'h' && b[0] == 'h')
5185                         instr->type = INSTR_JMP_GT_MH;
5186                 if (a[0] == 'h' && b[0] == 'h')
5187                         instr->type = INSTR_JMP_GT_HH;
5188                 instr->jmp.ip = NULL; /* Resolved later. */
5189
5190                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5191                 instr->jmp.a.n_bits = fa->n_bits;
5192                 instr->jmp.a.offset = fa->offset / 8;
5193                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5194                 instr->jmp.b.n_bits = fb->n_bits;
5195                 instr->jmp.b.offset = fb->offset / 8;
5196                 return 0;
5197         }
5198
5199         /* JMP_GT_MI, JMP_GT_HI. */
5200         b_val = strtoull(b, &b, 0);
5201         CHECK(!b[0], EINVAL);
5202
5203         instr->type = INSTR_JMP_GT_MI;
5204         if (a[0] == 'h')
5205                 instr->type = INSTR_JMP_GT_HI;
5206         instr->jmp.ip = NULL; /* Resolved later. */
5207
5208         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5209         instr->jmp.a.n_bits = fa->n_bits;
5210         instr->jmp.a.offset = fa->offset / 8;
5211         instr->jmp.b_val = b_val;
5212         return 0;
5213 }
5214
5215 static inline void
5216 instr_jmp_exec(struct rte_swx_pipeline *p)
5217 {
5218         struct thread *t = &p->threads[p->thread_id];
5219         struct instruction *ip = t->ip;
5220
5221         TRACE("[Thread %2u] jmp\n", p->thread_id);
5222
5223         thread_ip_set(t, ip->jmp.ip);
5224 }
5225
5226 static inline void
5227 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5228 {
5229         struct thread *t = &p->threads[p->thread_id];
5230         struct instruction *ip = t->ip;
5231         uint32_t header_id = ip->jmp.header_id;
5232
5233         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5234
5235         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5236 }
5237
5238 static inline void
5239 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5240 {
5241         struct thread *t = &p->threads[p->thread_id];
5242         struct instruction *ip = t->ip;
5243         uint32_t header_id = ip->jmp.header_id;
5244
5245         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5246
5247         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5248 }
5249
5250 static inline void
5251 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5252 {
5253         struct thread *t = &p->threads[p->thread_id];
5254         struct instruction *ip = t->ip;
5255         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5256
5257         TRACE("[Thread %2u] jmph\n", p->thread_id);
5258
5259         t->ip = ip_next[t->hit];
5260 }
5261
5262 static inline void
5263 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5264 {
5265         struct thread *t = &p->threads[p->thread_id];
5266         struct instruction *ip = t->ip;
5267         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5268
5269         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5270
5271         t->ip = ip_next[t->hit];
5272 }
5273
5274 static inline void
5275 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5276 {
5277         struct thread *t = &p->threads[p->thread_id];
5278         struct instruction *ip = t->ip;
5279
5280         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5281
5282         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5283 }
5284
5285 static inline void
5286 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5287 {
5288         struct thread *t = &p->threads[p->thread_id];
5289         struct instruction *ip = t->ip;
5290
5291         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5292
5293         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5294 }
5295
5296 static inline void
5297 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5298 {
5299         struct thread *t = &p->threads[p->thread_id];
5300         struct instruction *ip = t->ip;
5301
5302         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5303
5304         JMP_CMP(t, ip, ==);
5305 }
5306
5307 static inline void
5308 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5309 {
5310         struct thread *t = &p->threads[p->thread_id];
5311         struct instruction *ip = t->ip;
5312
5313         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5314
5315         JMP_CMP_MH(t, ip, ==);
5316 }
5317
5318 static inline void
5319 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5320 {
5321         struct thread *t = &p->threads[p->thread_id];
5322         struct instruction *ip = t->ip;
5323
5324         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5325
5326         JMP_CMP_HM(t, ip, ==);
5327 }
5328
5329 static inline void
5330 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5331 {
5332         struct thread *t = &p->threads[p->thread_id];
5333         struct instruction *ip = t->ip;
5334
5335         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5336
5337         JMP_CMP_HH_FAST(t, ip, ==);
5338 }
5339
5340 static inline void
5341 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5342 {
5343         struct thread *t = &p->threads[p->thread_id];
5344         struct instruction *ip = t->ip;
5345
5346         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5347
5348         JMP_CMP_I(t, ip, ==);
5349 }
5350
5351 static inline void
5352 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5353 {
5354         struct thread *t = &p->threads[p->thread_id];
5355         struct instruction *ip = t->ip;
5356
5357         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5358
5359         JMP_CMP(t, ip, !=);
5360 }
5361
5362 static inline void
5363 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5364 {
5365         struct thread *t = &p->threads[p->thread_id];
5366         struct instruction *ip = t->ip;
5367
5368         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5369
5370         JMP_CMP_MH(t, ip, !=);
5371 }
5372
5373 static inline void
5374 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5375 {
5376         struct thread *t = &p->threads[p->thread_id];
5377         struct instruction *ip = t->ip;
5378
5379         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5380
5381         JMP_CMP_HM(t, ip, !=);
5382 }
5383
5384 static inline void
5385 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5386 {
5387         struct thread *t = &p->threads[p->thread_id];
5388         struct instruction *ip = t->ip;
5389
5390         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5391
5392         JMP_CMP_HH_FAST(t, ip, !=);
5393 }
5394
5395 static inline void
5396 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5397 {
5398         struct thread *t = &p->threads[p->thread_id];
5399         struct instruction *ip = t->ip;
5400
5401         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5402
5403         JMP_CMP_I(t, ip, !=);
5404 }
5405
5406 static inline void
5407 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5408 {
5409         struct thread *t = &p->threads[p->thread_id];
5410         struct instruction *ip = t->ip;
5411
5412         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5413
5414         JMP_CMP(t, ip, <);
5415 }
5416
5417 static inline void
5418 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5419 {
5420         struct thread *t = &p->threads[p->thread_id];
5421         struct instruction *ip = t->ip;
5422
5423         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5424
5425         JMP_CMP_MH(t, ip, <);
5426 }
5427
5428 static inline void
5429 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5430 {
5431         struct thread *t = &p->threads[p->thread_id];
5432         struct instruction *ip = t->ip;
5433
5434         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5435
5436         JMP_CMP_HM(t, ip, <);
5437 }
5438
5439 static inline void
5440 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5441 {
5442         struct thread *t = &p->threads[p->thread_id];
5443         struct instruction *ip = t->ip;
5444
5445         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5446
5447         JMP_CMP_HH(t, ip, <);
5448 }
5449
5450 static inline void
5451 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5452 {
5453         struct thread *t = &p->threads[p->thread_id];
5454         struct instruction *ip = t->ip;
5455
5456         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5457
5458         JMP_CMP_MI(t, ip, <);
5459 }
5460
5461 static inline void
5462 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5463 {
5464         struct thread *t = &p->threads[p->thread_id];
5465         struct instruction *ip = t->ip;
5466
5467         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5468
5469         JMP_CMP_HI(t, ip, <);
5470 }
5471
5472 static inline void
5473 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5474 {
5475         struct thread *t = &p->threads[p->thread_id];
5476         struct instruction *ip = t->ip;
5477
5478         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5479
5480         JMP_CMP(t, ip, >);
5481 }
5482
5483 static inline void
5484 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5485 {
5486         struct thread *t = &p->threads[p->thread_id];
5487         struct instruction *ip = t->ip;
5488
5489         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5490
5491         JMP_CMP_MH(t, ip, >);
5492 }
5493
5494 static inline void
5495 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5496 {
5497         struct thread *t = &p->threads[p->thread_id];
5498         struct instruction *ip = t->ip;
5499
5500         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5501
5502         JMP_CMP_HM(t, ip, >);
5503 }
5504
5505 static inline void
5506 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5507 {
5508         struct thread *t = &p->threads[p->thread_id];
5509         struct instruction *ip = t->ip;
5510
5511         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5512
5513         JMP_CMP_HH(t, ip, >);
5514 }
5515
5516 static inline void
5517 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5518 {
5519         struct thread *t = &p->threads[p->thread_id];
5520         struct instruction *ip = t->ip;
5521
5522         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5523
5524         JMP_CMP_MI(t, ip, >);
5525 }
5526
5527 static inline void
5528 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5529 {
5530         struct thread *t = &p->threads[p->thread_id];
5531         struct instruction *ip = t->ip;
5532
5533         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5534
5535         JMP_CMP_HI(t, ip, >);
5536 }
5537
5538 /*
5539  * return.
5540  */
5541 static int
5542 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5543                        struct action *action,
5544                        char **tokens __rte_unused,
5545                        int n_tokens,
5546                        struct instruction *instr,
5547                        struct instruction_data *data __rte_unused)
5548 {
5549         CHECK(action, EINVAL);
5550         CHECK(n_tokens == 1, EINVAL);
5551
5552         instr->type = INSTR_RETURN;
5553         return 0;
5554 }
5555
5556 static inline void
5557 instr_return_exec(struct rte_swx_pipeline *p)
5558 {
5559         struct thread *t = &p->threads[p->thread_id];
5560
5561         TRACE("[Thread %2u] return\n", p->thread_id);
5562
5563         t->ip = t->ret;
5564 }
5565
5566 static int
5567 instr_translate(struct rte_swx_pipeline *p,
5568                 struct action *action,
5569                 char *string,
5570                 struct instruction *instr,
5571                 struct instruction_data *data)
5572 {
5573         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5574         int n_tokens = 0, tpos = 0;
5575
5576         /* Parse the instruction string into tokens. */
5577         for ( ; ; ) {
5578                 char *token;
5579
5580                 token = strtok_r(string, " \t\v", &string);
5581                 if (!token)
5582                         break;
5583
5584                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5585                 CHECK_NAME(token, EINVAL);
5586
5587                 tokens[n_tokens] = token;
5588                 n_tokens++;
5589         }
5590
5591         CHECK(n_tokens, EINVAL);
5592
5593         /* Handle the optional instruction label. */
5594         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5595                 strcpy(data->label, tokens[0]);
5596
5597                 tpos += 2;
5598                 CHECK(n_tokens - tpos, EINVAL);
5599         }
5600
5601         /* Identify the instruction type. */
5602         if (!strcmp(tokens[tpos], "rx"))
5603                 return instr_rx_translate(p,
5604                                           action,
5605                                           &tokens[tpos],
5606                                           n_tokens - tpos,
5607                                           instr,
5608                                           data);
5609
5610         if (!strcmp(tokens[tpos], "tx"))
5611                 return instr_tx_translate(p,
5612                                           action,
5613                                           &tokens[tpos],
5614                                           n_tokens - tpos,
5615                                           instr,
5616                                           data);
5617
5618         if (!strcmp(tokens[tpos], "drop"))
5619                 return instr_drop_translate(p,
5620                                             action,
5621                                             &tokens[tpos],
5622                                             n_tokens - tpos,
5623                                             instr,
5624                                             data);
5625
5626         if (!strcmp(tokens[tpos], "extract"))
5627                 return instr_hdr_extract_translate(p,
5628                                                    action,
5629                                                    &tokens[tpos],
5630                                                    n_tokens - tpos,
5631                                                    instr,
5632                                                    data);
5633
5634         if (!strcmp(tokens[tpos], "lookahead"))
5635                 return instr_hdr_lookahead_translate(p,
5636                                                      action,
5637                                                      &tokens[tpos],
5638                                                      n_tokens - tpos,
5639                                                      instr,
5640                                                      data);
5641
5642         if (!strcmp(tokens[tpos], "emit"))
5643                 return instr_hdr_emit_translate(p,
5644                                                 action,
5645                                                 &tokens[tpos],
5646                                                 n_tokens - tpos,
5647                                                 instr,
5648                                                 data);
5649
5650         if (!strcmp(tokens[tpos], "validate"))
5651                 return instr_hdr_validate_translate(p,
5652                                                     action,
5653                                                     &tokens[tpos],
5654                                                     n_tokens - tpos,
5655                                                     instr,
5656                                                     data);
5657
5658         if (!strcmp(tokens[tpos], "invalidate"))
5659                 return instr_hdr_invalidate_translate(p,
5660                                                       action,
5661                                                       &tokens[tpos],
5662                                                       n_tokens - tpos,
5663                                                       instr,
5664                                                       data);
5665
5666         if (!strcmp(tokens[tpos], "mov"))
5667                 return instr_mov_translate(p,
5668                                            action,
5669                                            &tokens[tpos],
5670                                            n_tokens - tpos,
5671                                            instr,
5672                                            data);
5673
5674         if (!strcmp(tokens[tpos], "add"))
5675                 return instr_alu_add_translate(p,
5676                                                action,
5677                                                &tokens[tpos],
5678                                                n_tokens - tpos,
5679                                                instr,
5680                                                data);
5681
5682         if (!strcmp(tokens[tpos], "sub"))
5683                 return instr_alu_sub_translate(p,
5684                                                action,
5685                                                &tokens[tpos],
5686                                                n_tokens - tpos,
5687                                                instr,
5688                                                data);
5689
5690         if (!strcmp(tokens[tpos], "ckadd"))
5691                 return instr_alu_ckadd_translate(p,
5692                                                  action,
5693                                                  &tokens[tpos],
5694                                                  n_tokens - tpos,
5695                                                  instr,
5696                                                  data);
5697
5698         if (!strcmp(tokens[tpos], "cksub"))
5699                 return instr_alu_cksub_translate(p,
5700                                                  action,
5701                                                  &tokens[tpos],
5702                                                  n_tokens - tpos,
5703                                                  instr,
5704                                                  data);
5705
5706         if (!strcmp(tokens[tpos], "and"))
5707                 return instr_alu_and_translate(p,
5708                                                action,
5709                                                &tokens[tpos],
5710                                                n_tokens - tpos,
5711                                                instr,
5712                                                data);
5713
5714         if (!strcmp(tokens[tpos], "or"))
5715                 return instr_alu_or_translate(p,
5716                                               action,
5717                                               &tokens[tpos],
5718                                               n_tokens - tpos,
5719                                               instr,
5720                                               data);
5721
5722         if (!strcmp(tokens[tpos], "xor"))
5723                 return instr_alu_xor_translate(p,
5724                                                action,
5725                                                &tokens[tpos],
5726                                                n_tokens - tpos,
5727                                                instr,
5728                                                data);
5729
5730         if (!strcmp(tokens[tpos], "shl"))
5731                 return instr_alu_shl_translate(p,
5732                                                action,
5733                                                &tokens[tpos],
5734                                                n_tokens - tpos,
5735                                                instr,
5736                                                data);
5737
5738         if (!strcmp(tokens[tpos], "shr"))
5739                 return instr_alu_shr_translate(p,
5740                                                action,
5741                                                &tokens[tpos],
5742                                                n_tokens - tpos,
5743                                                instr,
5744                                                data);
5745
5746         if (!strcmp(tokens[tpos], "regprefetch"))
5747                 return instr_regprefetch_translate(p,
5748                                                    action,
5749                                                    &tokens[tpos],
5750                                                    n_tokens - tpos,
5751                                                    instr,
5752                                                    data);
5753
5754         if (!strcmp(tokens[tpos], "regrd"))
5755                 return instr_regrd_translate(p,
5756                                              action,
5757                                              &tokens[tpos],
5758                                              n_tokens - tpos,
5759                                              instr,
5760                                              data);
5761
5762         if (!strcmp(tokens[tpos], "regwr"))
5763                 return instr_regwr_translate(p,
5764                                              action,
5765                                              &tokens[tpos],
5766                                              n_tokens - tpos,
5767                                              instr,
5768                                              data);
5769
5770         if (!strcmp(tokens[tpos], "regadd"))
5771                 return instr_regadd_translate(p,
5772                                               action,
5773                                               &tokens[tpos],
5774                                               n_tokens - tpos,
5775                                               instr,
5776                                               data);
5777
5778         if (!strcmp(tokens[tpos], "metprefetch"))
5779                 return instr_metprefetch_translate(p,
5780                                                    action,
5781                                                    &tokens[tpos],
5782                                                    n_tokens - tpos,
5783                                                    instr,
5784                                                    data);
5785
5786         if (!strcmp(tokens[tpos], "meter"))
5787                 return instr_meter_translate(p,
5788                                              action,
5789                                              &tokens[tpos],
5790                                              n_tokens - tpos,
5791                                              instr,
5792                                              data);
5793
5794         if (!strcmp(tokens[tpos], "table"))
5795                 return instr_table_translate(p,
5796                                              action,
5797                                              &tokens[tpos],
5798                                              n_tokens - tpos,
5799                                              instr,
5800                                              data);
5801
5802         if (!strcmp(tokens[tpos], "learn"))
5803                 return instr_learn_translate(p,
5804                                              action,
5805                                              &tokens[tpos],
5806                                              n_tokens - tpos,
5807                                              instr,
5808                                              data);
5809
5810         if (!strcmp(tokens[tpos], "forget"))
5811                 return instr_forget_translate(p,
5812                                               action,
5813                                               &tokens[tpos],
5814                                               n_tokens - tpos,
5815                                               instr,
5816                                               data);
5817
5818         if (!strcmp(tokens[tpos], "extern"))
5819                 return instr_extern_translate(p,
5820                                               action,
5821                                               &tokens[tpos],
5822                                               n_tokens - tpos,
5823                                               instr,
5824                                               data);
5825
5826         if (!strcmp(tokens[tpos], "jmp"))
5827                 return instr_jmp_translate(p,
5828                                            action,
5829                                            &tokens[tpos],
5830                                            n_tokens - tpos,
5831                                            instr,
5832                                            data);
5833
5834         if (!strcmp(tokens[tpos], "jmpv"))
5835                 return instr_jmp_valid_translate(p,
5836                                                  action,
5837                                                  &tokens[tpos],
5838                                                  n_tokens - tpos,
5839                                                  instr,
5840                                                  data);
5841
5842         if (!strcmp(tokens[tpos], "jmpnv"))
5843                 return instr_jmp_invalid_translate(p,
5844                                                    action,
5845                                                    &tokens[tpos],
5846                                                    n_tokens - tpos,
5847                                                    instr,
5848                                                    data);
5849
5850         if (!strcmp(tokens[tpos], "jmph"))
5851                 return instr_jmp_hit_translate(p,
5852                                                action,
5853                                                &tokens[tpos],
5854                                                n_tokens - tpos,
5855                                                instr,
5856                                                data);
5857
5858         if (!strcmp(tokens[tpos], "jmpnh"))
5859                 return instr_jmp_miss_translate(p,
5860                                                 action,
5861                                                 &tokens[tpos],
5862                                                 n_tokens - tpos,
5863                                                 instr,
5864                                                 data);
5865
5866         if (!strcmp(tokens[tpos], "jmpa"))
5867                 return instr_jmp_action_hit_translate(p,
5868                                                       action,
5869                                                       &tokens[tpos],
5870                                                       n_tokens - tpos,
5871                                                       instr,
5872                                                       data);
5873
5874         if (!strcmp(tokens[tpos], "jmpna"))
5875                 return instr_jmp_action_miss_translate(p,
5876                                                        action,
5877                                                        &tokens[tpos],
5878                                                        n_tokens - tpos,
5879                                                        instr,
5880                                                        data);
5881
5882         if (!strcmp(tokens[tpos], "jmpeq"))
5883                 return instr_jmp_eq_translate(p,
5884                                               action,
5885                                               &tokens[tpos],
5886                                               n_tokens - tpos,
5887                                               instr,
5888                                               data);
5889
5890         if (!strcmp(tokens[tpos], "jmpneq"))
5891                 return instr_jmp_neq_translate(p,
5892                                                action,
5893                                                &tokens[tpos],
5894                                                n_tokens - tpos,
5895                                                instr,
5896                                                data);
5897
5898         if (!strcmp(tokens[tpos], "jmplt"))
5899                 return instr_jmp_lt_translate(p,
5900                                               action,
5901                                               &tokens[tpos],
5902                                               n_tokens - tpos,
5903                                               instr,
5904                                               data);
5905
5906         if (!strcmp(tokens[tpos], "jmpgt"))
5907                 return instr_jmp_gt_translate(p,
5908                                               action,
5909                                               &tokens[tpos],
5910                                               n_tokens - tpos,
5911                                               instr,
5912                                               data);
5913
5914         if (!strcmp(tokens[tpos], "return"))
5915                 return instr_return_translate(p,
5916                                               action,
5917                                               &tokens[tpos],
5918                                               n_tokens - tpos,
5919                                               instr,
5920                                               data);
5921
5922         CHECK(0, EINVAL);
5923 }
5924
5925 static struct instruction_data *
5926 label_find(struct instruction_data *data, uint32_t n, const char *label)
5927 {
5928         uint32_t i;
5929
5930         for (i = 0; i < n; i++)
5931                 if (!strcmp(label, data[i].label))
5932                         return &data[i];
5933
5934         return NULL;
5935 }
5936
5937 static uint32_t
5938 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5939 {
5940         uint32_t count = 0, i;
5941
5942         if (!label[0])
5943                 return 0;
5944
5945         for (i = 0; i < n; i++)
5946                 if (!strcmp(label, data[i].jmp_label))
5947                         count++;
5948
5949         return count;
5950 }
5951
5952 static int
5953 instr_label_check(struct instruction_data *instruction_data,
5954                   uint32_t n_instructions)
5955 {
5956         uint32_t i;
5957
5958         /* Check that all instruction labels are unique. */
5959         for (i = 0; i < n_instructions; i++) {
5960                 struct instruction_data *data = &instruction_data[i];
5961                 char *label = data->label;
5962                 uint32_t j;
5963
5964                 if (!label[0])
5965                         continue;
5966
5967                 for (j = i + 1; j < n_instructions; j++)
5968                         CHECK(strcmp(label, data[j].label), EINVAL);
5969         }
5970
5971         /* Get users for each instruction label. */
5972         for (i = 0; i < n_instructions; i++) {
5973                 struct instruction_data *data = &instruction_data[i];
5974                 char *label = data->label;
5975
5976                 data->n_users = label_is_used(instruction_data,
5977                                               n_instructions,
5978                                               label);
5979         }
5980
5981         return 0;
5982 }
5983
5984 static int
5985 instr_jmp_resolve(struct instruction *instructions,
5986                   struct instruction_data *instruction_data,
5987                   uint32_t n_instructions)
5988 {
5989         uint32_t i;
5990
5991         for (i = 0; i < n_instructions; i++) {
5992                 struct instruction *instr = &instructions[i];
5993                 struct instruction_data *data = &instruction_data[i];
5994                 struct instruction_data *found;
5995
5996                 if (!instruction_is_jmp(instr))
5997                         continue;
5998
5999                 found = label_find(instruction_data,
6000                                    n_instructions,
6001                                    data->jmp_label);
6002                 CHECK(found, EINVAL);
6003
6004                 instr->jmp.ip = &instructions[found - instruction_data];
6005         }
6006
6007         return 0;
6008 }
6009
6010 static int
6011 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6012              struct action *a,
6013              struct instruction *instr,
6014              struct instruction_data *data __rte_unused,
6015              uint32_t n_instructions)
6016 {
6017         if (!a) {
6018                 enum instruction_type type;
6019                 uint32_t i;
6020
6021                 /* Check that the first instruction is rx. */
6022                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6023
6024                 /* Check that there is at least one tx instruction. */
6025                 for (i = 0; i < n_instructions; i++) {
6026                         type = instr[i].type;
6027
6028                         if (instruction_is_tx(type))
6029                                 break;
6030                 }
6031                 CHECK(i < n_instructions, EINVAL);
6032
6033                 /* Check that the last instruction is either tx or unconditional
6034                  * jump.
6035                  */
6036                 type = instr[n_instructions - 1].type;
6037                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6038         }
6039
6040         if (a) {
6041                 enum instruction_type type;
6042                 uint32_t i;
6043
6044                 /* Check that there is at least one return or tx instruction. */
6045                 for (i = 0; i < n_instructions; i++) {
6046                         type = instr[i].type;
6047
6048                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6049                                 break;
6050                 }
6051                 CHECK(i < n_instructions, EINVAL);
6052         }
6053
6054         return 0;
6055 }
6056
6057 static uint32_t
6058 instr_compact(struct instruction *instructions,
6059               struct instruction_data *instruction_data,
6060               uint32_t n_instructions)
6061 {
6062         uint32_t i, pos = 0;
6063
6064         /* Eliminate the invalid instructions that have been optimized out. */
6065         for (i = 0; i < n_instructions; i++) {
6066                 struct instruction *instr = &instructions[i];
6067                 struct instruction_data *data = &instruction_data[i];
6068
6069                 if (data->invalid)
6070                         continue;
6071
6072                 if (i != pos) {
6073                         memcpy(&instructions[pos], instr, sizeof(*instr));
6074                         memcpy(&instruction_data[pos], data, sizeof(*data));
6075                 }
6076
6077                 pos++;
6078         }
6079
6080         return pos;
6081 }
6082
6083 static int
6084 instr_pattern_extract_many_search(struct instruction *instr,
6085                                   struct instruction_data *data,
6086                                   uint32_t n_instr,
6087                                   uint32_t *n_pattern_instr)
6088 {
6089         uint32_t i;
6090
6091         for (i = 0; i < n_instr; i++) {
6092                 if (data[i].invalid)
6093                         break;
6094
6095                 if (instr[i].type != INSTR_HDR_EXTRACT)
6096                         break;
6097
6098                 if (i == RTE_DIM(instr->io.hdr.header_id))
6099                         break;
6100
6101                 if (i && data[i].n_users)
6102                         break;
6103         }
6104
6105         if (i < 2)
6106                 return 0;
6107
6108         *n_pattern_instr = i;
6109         return 1;
6110 }
6111
6112 static void
6113 instr_pattern_extract_many_replace(struct instruction *instr,
6114                                    struct instruction_data *data,
6115                                    uint32_t n_instr)
6116 {
6117         uint32_t i;
6118
6119         for (i = 1; i < n_instr; i++) {
6120                 instr[0].type++;
6121                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6122                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6123                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6124
6125                 data[i].invalid = 1;
6126         }
6127 }
6128
6129 static uint32_t
6130 instr_pattern_extract_many_optimize(struct instruction *instructions,
6131                                     struct instruction_data *instruction_data,
6132                                     uint32_t n_instructions)
6133 {
6134         uint32_t i;
6135
6136         for (i = 0; i < n_instructions; ) {
6137                 struct instruction *instr = &instructions[i];
6138                 struct instruction_data *data = &instruction_data[i];
6139                 uint32_t n_instr = 0;
6140                 int detected;
6141
6142                 /* Extract many. */
6143                 detected = instr_pattern_extract_many_search(instr,
6144                                                              data,
6145                                                              n_instructions - i,
6146                                                              &n_instr);
6147                 if (detected) {
6148                         instr_pattern_extract_many_replace(instr,
6149                                                            data,
6150                                                            n_instr);
6151                         i += n_instr;
6152                         continue;
6153                 }
6154
6155                 /* No pattern starting at the current instruction. */
6156                 i++;
6157         }
6158
6159         /* Eliminate the invalid instructions that have been optimized out. */
6160         n_instructions = instr_compact(instructions,
6161                                        instruction_data,
6162                                        n_instructions);
6163
6164         return n_instructions;
6165 }
6166
6167 static int
6168 instr_pattern_emit_many_tx_search(struct instruction *instr,
6169                                   struct instruction_data *data,
6170                                   uint32_t n_instr,
6171                                   uint32_t *n_pattern_instr)
6172 {
6173         uint32_t i;
6174
6175         for (i = 0; i < n_instr; i++) {
6176                 if (data[i].invalid)
6177                         break;
6178
6179                 if (instr[i].type != INSTR_HDR_EMIT)
6180                         break;
6181
6182                 if (i == RTE_DIM(instr->io.hdr.header_id))
6183                         break;
6184
6185                 if (i && data[i].n_users)
6186                         break;
6187         }
6188
6189         if (!i)
6190                 return 0;
6191
6192         if (!instruction_is_tx(instr[i].type))
6193                 return 0;
6194
6195         if (data[i].n_users)
6196                 return 0;
6197
6198         i++;
6199
6200         *n_pattern_instr = i;
6201         return 1;
6202 }
6203
6204 static void
6205 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6206                                    struct instruction_data *data,
6207                                    uint32_t n_instr)
6208 {
6209         uint32_t i;
6210
6211         /* Any emit instruction in addition to the first one. */
6212         for (i = 1; i < n_instr - 1; i++) {
6213                 instr[0].type++;
6214                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6215                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6216                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6217
6218                 data[i].invalid = 1;
6219         }
6220
6221         /* The TX instruction is the last one in the pattern. */
6222         instr[0].type++;
6223         instr[0].io.io.offset = instr[i].io.io.offset;
6224         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6225         data[i].invalid = 1;
6226 }
6227
6228 static uint32_t
6229 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6230                                     struct instruction_data *instruction_data,
6231                                     uint32_t n_instructions)
6232 {
6233         uint32_t i;
6234
6235         for (i = 0; i < n_instructions; ) {
6236                 struct instruction *instr = &instructions[i];
6237                 struct instruction_data *data = &instruction_data[i];
6238                 uint32_t n_instr = 0;
6239                 int detected;
6240
6241                 /* Emit many + TX. */
6242                 detected = instr_pattern_emit_many_tx_search(instr,
6243                                                              data,
6244                                                              n_instructions - i,
6245                                                              &n_instr);
6246                 if (detected) {
6247                         instr_pattern_emit_many_tx_replace(instr,
6248                                                            data,
6249                                                            n_instr);
6250                         i += n_instr;
6251                         continue;
6252                 }
6253
6254                 /* No pattern starting at the current instruction. */
6255                 i++;
6256         }
6257
6258         /* Eliminate the invalid instructions that have been optimized out. */
6259         n_instructions = instr_compact(instructions,
6260                                        instruction_data,
6261                                        n_instructions);
6262
6263         return n_instructions;
6264 }
6265
6266 static uint32_t
6267 action_arg_src_mov_count(struct action *a,
6268                          uint32_t arg_id,
6269                          struct instruction *instructions,
6270                          struct instruction_data *instruction_data,
6271                          uint32_t n_instructions);
6272
6273 static int
6274 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6275                                       struct action *a,
6276                                       struct instruction *instr,
6277                                       struct instruction_data *data,
6278                                       uint32_t n_instr,
6279                                       struct instruction *instructions,
6280                                       struct instruction_data *instruction_data,
6281                                       uint32_t n_instructions,
6282                                       uint32_t *n_pattern_instr)
6283 {
6284         struct header *h;
6285         uint32_t src_field_id, i, j;
6286
6287         /* Prerequisites. */
6288         if (!a || !a->st)
6289                 return 0;
6290
6291         /* First instruction: MOV_HM. */
6292         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6293                 return 0;
6294
6295         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6296         if (!h || h->st->var_size)
6297                 return 0;
6298
6299         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6300                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6301                         break;
6302
6303         if (src_field_id == a->st->n_fields)
6304                 return 0;
6305
6306         if (instr[0].mov.dst.offset ||
6307             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6308             instr[0].mov.src.struct_id ||
6309             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6310             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6311                 return 0;
6312
6313         if ((n_instr < h->st->n_fields + 1) ||
6314              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6315                 return 0;
6316
6317         /* Subsequent instructions: MOV_HM. */
6318         for (i = 1; i < h->st->n_fields; i++)
6319                 if (data[i].invalid ||
6320                     data[i].n_users ||
6321                     (instr[i].type != INSTR_MOV_HM) ||
6322                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6323                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6324                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6325                     instr[i].mov.src.struct_id ||
6326                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6327                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6328                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6329                         return 0;
6330
6331         /* Last instruction: HDR_VALIDATE. */
6332         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6333             (instr[i].valid.header_id != h->id))
6334                 return 0;
6335
6336         /* Check that none of the action args that are used as source for this
6337          * DMA transfer are not used as source in any other mov instruction.
6338          */
6339         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6340                 uint32_t n_users;
6341
6342                 n_users = action_arg_src_mov_count(a,
6343                                                    j,
6344                                                    instructions,
6345                                                    instruction_data,
6346                                                    n_instructions);
6347                 if (n_users > 1)
6348                         return 0;
6349         }
6350
6351         *n_pattern_instr = 1 + i;
6352         return 1;
6353 }
6354
6355 static void
6356 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6357                                        struct action *a,
6358                                        struct instruction *instr,
6359                                        struct instruction_data *data,
6360                                        uint32_t n_instr)
6361 {
6362         struct header *h;
6363         uint32_t src_field_id, src_offset, i;
6364
6365         /* Read from the instructions before they are modified. */
6366         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6367         if (!h)
6368                 return;
6369
6370         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6371                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6372                         break;
6373
6374         if (src_field_id == a->st->n_fields)
6375                 return;
6376
6377         src_offset = instr[0].mov.src.offset;
6378
6379         /* Modify the instructions. */
6380         instr[0].type = INSTR_DMA_HT;
6381         instr[0].dma.dst.header_id[0] = h->id;
6382         instr[0].dma.dst.struct_id[0] = h->struct_id;
6383         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6384         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6385
6386         for (i = 1; i < n_instr; i++)
6387                 data[i].invalid = 1;
6388
6389         /* Update the endianness of the action arguments to header endianness. */
6390         for (i = 0; i < h->st->n_fields; i++)
6391                 a->args_endianness[src_field_id + i] = 1;
6392 }
6393
6394 static uint32_t
6395 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6396                                         struct action *a,
6397                                         struct instruction *instructions,
6398                                         struct instruction_data *instruction_data,
6399                                         uint32_t n_instructions)
6400 {
6401         uint32_t i;
6402
6403         if (!a || !a->st)
6404                 return n_instructions;
6405
6406         for (i = 0; i < n_instructions; ) {
6407                 struct instruction *instr = &instructions[i];
6408                 struct instruction_data *data = &instruction_data[i];
6409                 uint32_t n_instr = 0;
6410                 int detected;
6411
6412                 /* Mov all + validate. */
6413                 detected = instr_pattern_mov_all_validate_search(p,
6414                                                                  a,
6415                                                                  instr,
6416                                                                  data,
6417                                                                  n_instructions - i,
6418                                                                  instructions,
6419                                                                  instruction_data,
6420                                                                  n_instructions,
6421                                                                  &n_instr);
6422                 if (detected) {
6423                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6424                         i += n_instr;
6425                         continue;
6426                 }
6427
6428                 /* No pattern starting at the current instruction. */
6429                 i++;
6430         }
6431
6432         /* Eliminate the invalid instructions that have been optimized out. */
6433         n_instructions = instr_compact(instructions,
6434                                        instruction_data,
6435                                        n_instructions);
6436
6437         return n_instructions;
6438 }
6439
6440 static int
6441 instr_pattern_dma_many_search(struct instruction *instr,
6442                               struct instruction_data *data,
6443                               uint32_t n_instr,
6444                               uint32_t *n_pattern_instr)
6445 {
6446         uint32_t i;
6447
6448         for (i = 0; i < n_instr; i++) {
6449                 if (data[i].invalid)
6450                         break;
6451
6452                 if (instr[i].type != INSTR_DMA_HT)
6453                         break;
6454
6455                 if (i == RTE_DIM(instr->dma.dst.header_id))
6456                         break;
6457
6458                 if (i && data[i].n_users)
6459                         break;
6460         }
6461
6462         if (i < 2)
6463                 return 0;
6464
6465         *n_pattern_instr = i;
6466         return 1;
6467 }
6468
6469 static void
6470 instr_pattern_dma_many_replace(struct instruction *instr,
6471                                struct instruction_data *data,
6472                                uint32_t n_instr)
6473 {
6474         uint32_t i;
6475
6476         for (i = 1; i < n_instr; i++) {
6477                 instr[0].type++;
6478                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6479                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6480                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6481                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6482
6483                 data[i].invalid = 1;
6484         }
6485 }
6486
6487 static uint32_t
6488 instr_pattern_dma_many_optimize(struct instruction *instructions,
6489                struct instruction_data *instruction_data,
6490                uint32_t n_instructions)
6491 {
6492         uint32_t i;
6493
6494         for (i = 0; i < n_instructions; ) {
6495                 struct instruction *instr = &instructions[i];
6496                 struct instruction_data *data = &instruction_data[i];
6497                 uint32_t n_instr = 0;
6498                 int detected;
6499
6500                 /* DMA many. */
6501                 detected = instr_pattern_dma_many_search(instr,
6502                                                          data,
6503                                                          n_instructions - i,
6504                                                          &n_instr);
6505                 if (detected) {
6506                         instr_pattern_dma_many_replace(instr, data, n_instr);
6507                         i += n_instr;
6508                         continue;
6509                 }
6510
6511                 /* No pattern starting at the current instruction. */
6512                 i++;
6513         }
6514
6515         /* Eliminate the invalid instructions that have been optimized out. */
6516         n_instructions = instr_compact(instructions,
6517                                        instruction_data,
6518                                        n_instructions);
6519
6520         return n_instructions;
6521 }
6522
6523 static uint32_t
6524 instr_optimize(struct rte_swx_pipeline *p,
6525                struct action *a,
6526                struct instruction *instructions,
6527                struct instruction_data *instruction_data,
6528                uint32_t n_instructions)
6529 {
6530         /* Extract many. */
6531         n_instructions = instr_pattern_extract_many_optimize(instructions,
6532                                                              instruction_data,
6533                                                              n_instructions);
6534
6535         /* Emit many + TX. */
6536         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6537                                                              instruction_data,
6538                                                              n_instructions);
6539
6540         /* Mov all + validate. */
6541         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6542                                                                  a,
6543                                                                  instructions,
6544                                                                  instruction_data,
6545                                                                  n_instructions);
6546
6547         /* DMA many. */
6548         n_instructions = instr_pattern_dma_many_optimize(instructions,
6549                                                          instruction_data,
6550                                                          n_instructions);
6551
6552         return n_instructions;
6553 }
6554
6555 static int
6556 instruction_config(struct rte_swx_pipeline *p,
6557                    struct action *a,
6558                    const char **instructions,
6559                    uint32_t n_instructions)
6560 {
6561         struct instruction *instr = NULL;
6562         struct instruction_data *data = NULL;
6563         int err = 0;
6564         uint32_t i;
6565
6566         CHECK(n_instructions, EINVAL);
6567         CHECK(instructions, EINVAL);
6568         for (i = 0; i < n_instructions; i++)
6569                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6570
6571         /* Memory allocation. */
6572         instr = calloc(n_instructions, sizeof(struct instruction));
6573         if (!instr) {
6574                 err = -ENOMEM;
6575                 goto error;
6576         }
6577
6578         data = calloc(n_instructions, sizeof(struct instruction_data));
6579         if (!data) {
6580                 err = -ENOMEM;
6581                 goto error;
6582         }
6583
6584         for (i = 0; i < n_instructions; i++) {
6585                 char *string = strdup(instructions[i]);
6586                 if (!string) {
6587                         err = -ENOMEM;
6588                         goto error;
6589                 }
6590
6591                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6592                 if (err) {
6593                         free(string);
6594                         goto error;
6595                 }
6596
6597                 free(string);
6598         }
6599
6600         err = instr_label_check(data, n_instructions);
6601         if (err)
6602                 goto error;
6603
6604         err = instr_verify(p, a, instr, data, n_instructions);
6605         if (err)
6606                 goto error;
6607
6608         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6609
6610         err = instr_jmp_resolve(instr, data, n_instructions);
6611         if (err)
6612                 goto error;
6613
6614         if (a) {
6615                 a->instructions = instr;
6616                 a->instruction_data = data;
6617                 a->n_instructions = n_instructions;
6618         } else {
6619                 p->instructions = instr;
6620                 p->instruction_data = data;
6621                 p->n_instructions = n_instructions;
6622         }
6623
6624         return 0;
6625
6626 error:
6627         free(data);
6628         free(instr);
6629         return err;
6630 }
6631
6632 static instr_exec_t instruction_table[] = {
6633         [INSTR_RX] = instr_rx_exec,
6634         [INSTR_TX] = instr_tx_exec,
6635         [INSTR_TX_I] = instr_tx_i_exec,
6636
6637         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6638         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6639         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6640         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6641         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6642         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6643         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6644         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6645         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6646         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6647
6648         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6649         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6650         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6651         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6652         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6653         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6654         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6655         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6656         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6657
6658         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6659         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6660
6661         [INSTR_MOV] = instr_mov_exec,
6662         [INSTR_MOV_MH] = instr_mov_mh_exec,
6663         [INSTR_MOV_HM] = instr_mov_hm_exec,
6664         [INSTR_MOV_HH] = instr_mov_hh_exec,
6665         [INSTR_MOV_I] = instr_mov_i_exec,
6666
6667         [INSTR_DMA_HT] = instr_dma_ht_exec,
6668         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6669         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6670         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6671         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6672         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6673         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6674         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6675
6676         [INSTR_ALU_ADD] = instr_alu_add_exec,
6677         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6678         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6679         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6680         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6681         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6682
6683         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6684         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6685         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6686         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6687         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6688         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6689
6690         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6691         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6692         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6693         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6694
6695         [INSTR_ALU_AND] = instr_alu_and_exec,
6696         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6697         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6698         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6699         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6700
6701         [INSTR_ALU_OR] = instr_alu_or_exec,
6702         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6703         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6704         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6705         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6706
6707         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6708         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6709         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6710         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6711         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6712
6713         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6714         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6715         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6716         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6717         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6718         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6719
6720         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6721         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6722         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6723         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6724         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6725         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6726
6727         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6728         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6729         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6730
6731         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6732         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6733         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6734         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6735         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6736         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6737
6738         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6739         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6740         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6741         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6742         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6743         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6744         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6745         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6746         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6747
6748         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6749         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6750         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6751         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6752         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6753         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6754         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6755         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6756         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6757
6758         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6759         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6760         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6761
6762         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6763         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6764         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6765         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6766         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6767         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6768         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6769         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6770         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6771         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6772         [INSTR_METER_IMM] = instr_meter_imm_exec,
6773         [INSTR_METER_IMI] = instr_meter_imi_exec,
6774
6775         [INSTR_TABLE] = instr_table_exec,
6776         [INSTR_TABLE_AF] = instr_table_af_exec,
6777         [INSTR_SELECTOR] = instr_selector_exec,
6778         [INSTR_LEARNER] = instr_learner_exec,
6779         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6780         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6781         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6782         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6783         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6784
6785         [INSTR_JMP] = instr_jmp_exec,
6786         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6787         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6788         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6789         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6790         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6791         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6792
6793         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6794         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6795         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6796         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6797         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6798
6799         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6800         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6801         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6802         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6803         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6804
6805         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6806         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6807         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6808         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6809         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6810         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6811
6812         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6813         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6814         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6815         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6816         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6817         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6818
6819         [INSTR_RETURN] = instr_return_exec,
6820 };
6821
6822 static int
6823 instruction_table_build(struct rte_swx_pipeline *p)
6824 {
6825         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6826                                       sizeof(struct instr_exec_t *));
6827         if (!p->instruction_table)
6828                 return -EINVAL;
6829
6830         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6831
6832         return 0;
6833 }
6834
6835 static void
6836 instruction_table_build_free(struct rte_swx_pipeline *p)
6837 {
6838         if (!p->instruction_table)
6839                 return;
6840
6841         free(p->instruction_table);
6842         p->instruction_table = NULL;
6843 }
6844
6845 static void
6846 instruction_table_free(struct rte_swx_pipeline *p)
6847 {
6848         instruction_table_build_free(p);
6849 }
6850
6851 static inline void
6852 instr_exec(struct rte_swx_pipeline *p)
6853 {
6854         struct thread *t = &p->threads[p->thread_id];
6855         struct instruction *ip = t->ip;
6856         instr_exec_t instr = p->instruction_table[ip->type];
6857
6858         instr(p);
6859 }
6860
6861 /*
6862  * Action.
6863  */
6864 static struct action *
6865 action_find(struct rte_swx_pipeline *p, const char *name)
6866 {
6867         struct action *elem;
6868
6869         if (!name)
6870                 return NULL;
6871
6872         TAILQ_FOREACH(elem, &p->actions, node)
6873                 if (strcmp(elem->name, name) == 0)
6874                         return elem;
6875
6876         return NULL;
6877 }
6878
6879 static struct action *
6880 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6881 {
6882         struct action *action = NULL;
6883
6884         TAILQ_FOREACH(action, &p->actions, node)
6885                 if (action->id == id)
6886                         return action;
6887
6888         return NULL;
6889 }
6890
6891 static struct field *
6892 action_field_find(struct action *a, const char *name)
6893 {
6894         return a->st ? struct_type_field_find(a->st, name) : NULL;
6895 }
6896
6897 static struct field *
6898 action_field_parse(struct action *action, const char *name)
6899 {
6900         if (name[0] != 't' || name[1] != '.')
6901                 return NULL;
6902
6903         return action_field_find(action, &name[2]);
6904 }
6905
6906 static int
6907 action_has_nbo_args(struct action *a)
6908 {
6909         uint32_t i;
6910
6911         /* Return if the action does not have any args. */
6912         if (!a->st)
6913                 return 0; /* FALSE */
6914
6915         for (i = 0; i < a->st->n_fields; i++)
6916                 if (a->args_endianness[i])
6917                         return 1; /* TRUE */
6918
6919         return 0; /* FALSE */
6920 }
6921
6922 static int
6923 action_does_learning(struct action *a)
6924 {
6925         uint32_t i;
6926
6927         for (i = 0; i < a->n_instructions; i++)
6928                 switch (a->instructions[i].type) {
6929                 case INSTR_LEARNER_LEARN:
6930                         return 1; /* TRUE */
6931
6932                 case INSTR_LEARNER_FORGET:
6933                         return 1; /* TRUE */
6934
6935                 default:
6936                         continue;
6937                 }
6938
6939         return 0; /* FALSE */
6940 }
6941
6942 int
6943 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6944                                const char *name,
6945                                const char *args_struct_type_name,
6946                                const char **instructions,
6947                                uint32_t n_instructions)
6948 {
6949         struct struct_type *args_struct_type = NULL;
6950         struct action *a;
6951         int err;
6952
6953         CHECK(p, EINVAL);
6954
6955         CHECK_NAME(name, EINVAL);
6956         CHECK(!action_find(p, name), EEXIST);
6957
6958         if (args_struct_type_name) {
6959                 CHECK_NAME(args_struct_type_name, EINVAL);
6960                 args_struct_type = struct_type_find(p, args_struct_type_name);
6961                 CHECK(args_struct_type, EINVAL);
6962                 CHECK(!args_struct_type->var_size, EINVAL);
6963         }
6964
6965         /* Node allocation. */
6966         a = calloc(1, sizeof(struct action));
6967         CHECK(a, ENOMEM);
6968         if (args_struct_type) {
6969                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6970                 if (!a->args_endianness) {
6971                         free(a);
6972                         CHECK(0, ENOMEM);
6973                 }
6974         }
6975
6976         /* Node initialization. */
6977         strcpy(a->name, name);
6978         a->st = args_struct_type;
6979         a->id = p->n_actions;
6980
6981         /* Instruction translation. */
6982         err = instruction_config(p, a, instructions, n_instructions);
6983         if (err) {
6984                 free(a->args_endianness);
6985                 free(a);
6986                 return err;
6987         }
6988
6989         /* Node add to tailq. */
6990         TAILQ_INSERT_TAIL(&p->actions, a, node);
6991         p->n_actions++;
6992
6993         return 0;
6994 }
6995
6996 static int
6997 action_build(struct rte_swx_pipeline *p)
6998 {
6999         struct action *action;
7000
7001         /* p->action_instructions. */
7002         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7003         CHECK(p->action_instructions, ENOMEM);
7004
7005         TAILQ_FOREACH(action, &p->actions, node)
7006                 p->action_instructions[action->id] = action->instructions;
7007
7008         /* p->action_funcs. */
7009         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7010         CHECK(p->action_funcs, ENOMEM);
7011
7012         return 0;
7013 }
7014
7015 static void
7016 action_build_free(struct rte_swx_pipeline *p)
7017 {
7018         free(p->action_funcs);
7019         p->action_funcs = NULL;
7020
7021         free(p->action_instructions);
7022         p->action_instructions = NULL;
7023 }
7024
7025 static void
7026 action_free(struct rte_swx_pipeline *p)
7027 {
7028         action_build_free(p);
7029
7030         for ( ; ; ) {
7031                 struct action *action;
7032
7033                 action = TAILQ_FIRST(&p->actions);
7034                 if (!action)
7035                         break;
7036
7037                 TAILQ_REMOVE(&p->actions, action, node);
7038                 free(action->instruction_data);
7039                 free(action->instructions);
7040                 free(action);
7041         }
7042 }
7043
7044 static uint32_t
7045 action_arg_src_mov_count(struct action *a,
7046                          uint32_t arg_id,
7047                          struct instruction *instructions,
7048                          struct instruction_data *instruction_data,
7049                          uint32_t n_instructions)
7050 {
7051         uint32_t offset, n_users = 0, i;
7052
7053         if (!a->st ||
7054             (arg_id >= a->st->n_fields) ||
7055             !instructions ||
7056             !instruction_data ||
7057             !n_instructions)
7058                 return 0;
7059
7060         offset = a->st->fields[arg_id].offset / 8;
7061
7062         for (i = 0; i < n_instructions; i++) {
7063                 struct instruction *instr = &instructions[i];
7064                 struct instruction_data *data = &instruction_data[i];
7065
7066                 if (data->invalid ||
7067                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7068                     instr->mov.src.struct_id ||
7069                     (instr->mov.src.offset != offset))
7070                         continue;
7071
7072                 n_users++;
7073         }
7074
7075         return n_users;
7076 }
7077
7078 /*
7079  * Table.
7080  */
7081 static struct table_type *
7082 table_type_find(struct rte_swx_pipeline *p, const char *name)
7083 {
7084         struct table_type *elem;
7085
7086         TAILQ_FOREACH(elem, &p->table_types, node)
7087                 if (strcmp(elem->name, name) == 0)
7088                         return elem;
7089
7090         return NULL;
7091 }
7092
7093 static struct table_type *
7094 table_type_resolve(struct rte_swx_pipeline *p,
7095                    const char *recommended_type_name,
7096                    enum rte_swx_table_match_type match_type)
7097 {
7098         struct table_type *elem;
7099
7100         /* Only consider the recommended type if the match type is correct. */
7101         if (recommended_type_name)
7102                 TAILQ_FOREACH(elem, &p->table_types, node)
7103                         if (!strcmp(elem->name, recommended_type_name) &&
7104                             (elem->match_type == match_type))
7105                                 return elem;
7106
7107         /* Ignore the recommended type and get the first element with this match
7108          * type.
7109          */
7110         TAILQ_FOREACH(elem, &p->table_types, node)
7111                 if (elem->match_type == match_type)
7112                         return elem;
7113
7114         return NULL;
7115 }
7116
7117 static struct table *
7118 table_find(struct rte_swx_pipeline *p, const char *name)
7119 {
7120         struct table *elem;
7121
7122         TAILQ_FOREACH(elem, &p->tables, node)
7123                 if (strcmp(elem->name, name) == 0)
7124                         return elem;
7125
7126         return NULL;
7127 }
7128
7129 static struct table *
7130 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7131 {
7132         struct table *table = NULL;
7133
7134         TAILQ_FOREACH(table, &p->tables, node)
7135                 if (table->id == id)
7136                         return table;
7137
7138         return NULL;
7139 }
7140
7141 int
7142 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7143                                      const char *name,
7144                                      enum rte_swx_table_match_type match_type,
7145                                      struct rte_swx_table_ops *ops)
7146 {
7147         struct table_type *elem;
7148
7149         CHECK(p, EINVAL);
7150
7151         CHECK_NAME(name, EINVAL);
7152         CHECK(!table_type_find(p, name), EEXIST);
7153
7154         CHECK(ops, EINVAL);
7155         CHECK(ops->create, EINVAL);
7156         CHECK(ops->lkp, EINVAL);
7157         CHECK(ops->free, EINVAL);
7158
7159         /* Node allocation. */
7160         elem = calloc(1, sizeof(struct table_type));
7161         CHECK(elem, ENOMEM);
7162
7163         /* Node initialization. */
7164         strcpy(elem->name, name);
7165         elem->match_type = match_type;
7166         memcpy(&elem->ops, ops, sizeof(*ops));
7167
7168         /* Node add to tailq. */
7169         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7170
7171         return 0;
7172 }
7173
7174 static int
7175 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7176                          uint32_t n_fields,
7177                          enum rte_swx_table_match_type *match_type)
7178 {
7179         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7180
7181         for (i = 0; i < n_fields; i++) {
7182                 struct rte_swx_match_field_params  *f = &fields[i];
7183
7184                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7185                         n_fields_em++;
7186
7187                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7188                         n_fields_lpm++;
7189         }
7190
7191         if ((n_fields_lpm > 1) ||
7192             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7193                 return -EINVAL;
7194
7195         *match_type = (n_fields_em == n_fields) ?
7196                        RTE_SWX_TABLE_MATCH_EXACT :
7197                        RTE_SWX_TABLE_MATCH_WILDCARD;
7198
7199         return 0;
7200 }
7201
7202 static int
7203 table_match_fields_check(struct rte_swx_pipeline *p,
7204                          struct rte_swx_pipeline_table_params *params,
7205                          struct header **header)
7206 {
7207         struct header *h0 = NULL;
7208         struct field *hf, *mf;
7209         uint32_t *offset = NULL, i;
7210         int status = 0;
7211
7212         /* Return if no match fields. */
7213         if (!params->n_fields) {
7214                 if (params->fields) {
7215                         status = -EINVAL;
7216                         goto end;
7217                 }
7218
7219                 if (header)
7220                         *header = NULL;
7221
7222                 return 0;
7223         }
7224
7225         /* Memory allocation. */
7226         offset = calloc(params->n_fields, sizeof(uint32_t));
7227         if (!offset) {
7228                 status = -ENOMEM;
7229                 goto end;
7230         }
7231
7232         /* Check that all the match fields belong to either the same header or
7233          * to the meta-data.
7234          */
7235         hf = header_field_parse(p, params->fields[0].name, &h0);
7236         mf = metadata_field_parse(p, params->fields[0].name);
7237         if ((!hf && !mf) || (hf && hf->var_size)) {
7238                 status = -EINVAL;
7239                 goto end;
7240         }
7241
7242         offset[0] = h0 ? hf->offset : mf->offset;
7243
7244         for (i = 1; i < params->n_fields; i++)
7245                 if (h0) {
7246                         struct header *h;
7247
7248                         hf = header_field_parse(p, params->fields[i].name, &h);
7249                         if (!hf || (h->id != h0->id) || hf->var_size) {
7250                                 status = -EINVAL;
7251                                 goto end;
7252                         }
7253
7254                         offset[i] = hf->offset;
7255                 } else {
7256                         mf = metadata_field_parse(p, params->fields[i].name);
7257                         if (!mf) {
7258                                 status = -EINVAL;
7259                                 goto end;
7260                         }
7261
7262                         offset[i] = mf->offset;
7263                 }
7264
7265         /* Check that there are no duplicated match fields. */
7266         for (i = 0; i < params->n_fields; i++) {
7267                 uint32_t j;
7268
7269                 for (j = 0; j < i; j++)
7270                         if (offset[j] == offset[i]) {
7271                                 status = -EINVAL;
7272                                 goto end;
7273                         }
7274         }
7275
7276         /* Return. */
7277         if (header)
7278                 *header = h0;
7279
7280 end:
7281         free(offset);
7282         return status;
7283 }
7284
7285 int
7286 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7287                               const char *name,
7288                               struct rte_swx_pipeline_table_params *params,
7289                               const char *recommended_table_type_name,
7290                               const char *args,
7291                               uint32_t size)
7292 {
7293         struct table_type *type;
7294         struct table *t;
7295         struct action *default_action;
7296         struct header *header = NULL;
7297         uint32_t action_data_size_max = 0, i;
7298         int status = 0;
7299
7300         CHECK(p, EINVAL);
7301
7302         CHECK_NAME(name, EINVAL);
7303         CHECK(!table_find(p, name), EEXIST);
7304         CHECK(!selector_find(p, name), EEXIST);
7305         CHECK(!learner_find(p, name), EEXIST);
7306
7307         CHECK(params, EINVAL);
7308
7309         /* Match checks. */
7310         status = table_match_fields_check(p, params, &header);
7311         if (status)
7312                 return status;
7313
7314         /* Action checks. */
7315         CHECK(params->n_actions, EINVAL);
7316         CHECK(params->action_names, EINVAL);
7317         for (i = 0; i < params->n_actions; i++) {
7318                 const char *action_name = params->action_names[i];
7319                 struct action *a;
7320                 uint32_t action_data_size;
7321
7322                 CHECK_NAME(action_name, EINVAL);
7323
7324                 a = action_find(p, action_name);
7325                 CHECK(a, EINVAL);
7326                 CHECK(!action_does_learning(a), EINVAL);
7327
7328                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7329                 if (action_data_size > action_data_size_max)
7330                         action_data_size_max = action_data_size;
7331         }
7332
7333         CHECK_NAME(params->default_action_name, EINVAL);
7334         for (i = 0; i < p->n_actions; i++)
7335                 if (!strcmp(params->action_names[i],
7336                             params->default_action_name))
7337                         break;
7338         CHECK(i < params->n_actions, EINVAL);
7339         default_action = action_find(p, params->default_action_name);
7340         CHECK((default_action->st && params->default_action_data) ||
7341               !params->default_action_data, EINVAL);
7342
7343         /* Table type checks. */
7344         if (recommended_table_type_name)
7345                 CHECK_NAME(recommended_table_type_name, EINVAL);
7346
7347         if (params->n_fields) {
7348                 enum rte_swx_table_match_type match_type;
7349
7350                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7351                 if (status)
7352                         return status;
7353
7354                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7355                 CHECK(type, EINVAL);
7356         } else {
7357                 type = NULL;
7358         }
7359
7360         /* Memory allocation. */
7361         t = calloc(1, sizeof(struct table));
7362         CHECK(t, ENOMEM);
7363
7364         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7365         if (!t->fields) {
7366                 free(t);
7367                 CHECK(0, ENOMEM);
7368         }
7369
7370         t->actions = calloc(params->n_actions, sizeof(struct action *));
7371         if (!t->actions) {
7372                 free(t->fields);
7373                 free(t);
7374                 CHECK(0, ENOMEM);
7375         }
7376
7377         if (action_data_size_max) {
7378                 t->default_action_data = calloc(1, action_data_size_max);
7379                 if (!t->default_action_data) {
7380                         free(t->actions);
7381                         free(t->fields);
7382                         free(t);
7383                         CHECK(0, ENOMEM);
7384                 }
7385         }
7386
7387         /* Node initialization. */
7388         strcpy(t->name, name);
7389         if (args && args[0])
7390                 strcpy(t->args, args);
7391         t->type = type;
7392
7393         for (i = 0; i < params->n_fields; i++) {
7394                 struct rte_swx_match_field_params *field = &params->fields[i];
7395                 struct match_field *f = &t->fields[i];
7396
7397                 f->match_type = field->match_type;
7398                 f->field = header ?
7399                         header_field_parse(p, field->name, NULL) :
7400                         metadata_field_parse(p, field->name);
7401         }
7402         t->n_fields = params->n_fields;
7403         t->header = header;
7404
7405         for (i = 0; i < params->n_actions; i++)
7406                 t->actions[i] = action_find(p, params->action_names[i]);
7407         t->default_action = default_action;
7408         if (default_action->st)
7409                 memcpy(t->default_action_data,
7410                        params->default_action_data,
7411                        default_action->st->n_bits / 8);
7412         t->n_actions = params->n_actions;
7413         t->default_action_is_const = params->default_action_is_const;
7414         t->action_data_size_max = action_data_size_max;
7415
7416         t->size = size;
7417         t->id = p->n_tables;
7418
7419         /* Node add to tailq. */
7420         TAILQ_INSERT_TAIL(&p->tables, t, node);
7421         p->n_tables++;
7422
7423         return 0;
7424 }
7425
7426 static struct rte_swx_table_params *
7427 table_params_get(struct table *table)
7428 {
7429         struct rte_swx_table_params *params;
7430         struct field *first, *last;
7431         uint8_t *key_mask;
7432         uint32_t key_size, key_offset, action_data_size, i;
7433
7434         /* Memory allocation. */
7435         params = calloc(1, sizeof(struct rte_swx_table_params));
7436         if (!params)
7437                 return NULL;
7438
7439         /* Find first (smallest offset) and last (biggest offset) match fields. */
7440         first = table->fields[0].field;
7441         last = table->fields[0].field;
7442
7443         for (i = 0; i < table->n_fields; i++) {
7444                 struct field *f = table->fields[i].field;
7445
7446                 if (f->offset < first->offset)
7447                         first = f;
7448
7449                 if (f->offset > last->offset)
7450                         last = f;
7451         }
7452
7453         /* Key offset and size. */
7454         key_offset = first->offset / 8;
7455         key_size = (last->offset + last->n_bits - first->offset) / 8;
7456
7457         /* Memory allocation. */
7458         key_mask = calloc(1, key_size);
7459         if (!key_mask) {
7460                 free(params);
7461                 return NULL;
7462         }
7463
7464         /* Key mask. */
7465         for (i = 0; i < table->n_fields; i++) {
7466                 struct field *f = table->fields[i].field;
7467                 uint32_t start = (f->offset - first->offset) / 8;
7468                 size_t size = f->n_bits / 8;
7469
7470                 memset(&key_mask[start], 0xFF, size);
7471         }
7472
7473         /* Action data size. */
7474         action_data_size = 0;
7475         for (i = 0; i < table->n_actions; i++) {
7476                 struct action *action = table->actions[i];
7477                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7478
7479                 if (ads > action_data_size)
7480                         action_data_size = ads;
7481         }
7482
7483         /* Fill in. */
7484         params->match_type = table->type->match_type;
7485         params->key_size = key_size;
7486         params->key_offset = key_offset;
7487         params->key_mask0 = key_mask;
7488         params->action_data_size = action_data_size;
7489         params->n_keys_max = table->size;
7490
7491         return params;
7492 }
7493
7494 static void
7495 table_params_free(struct rte_swx_table_params *params)
7496 {
7497         if (!params)
7498                 return;
7499
7500         free(params->key_mask0);
7501         free(params);
7502 }
7503
7504 static int
7505 table_stub_lkp(void *table __rte_unused,
7506                void *mailbox __rte_unused,
7507                uint8_t **key __rte_unused,
7508                uint64_t *action_id __rte_unused,
7509                uint8_t **action_data __rte_unused,
7510                int *hit)
7511 {
7512         *hit = 0;
7513         return 1; /* DONE. */
7514 }
7515
7516 static int
7517 table_build(struct rte_swx_pipeline *p)
7518 {
7519         uint32_t i;
7520
7521         /* Per pipeline: table statistics. */
7522         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7523         CHECK(p->table_stats, ENOMEM);
7524
7525         for (i = 0; i < p->n_tables; i++) {
7526                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7527                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7528         }
7529
7530         /* Per thread: table runt-time. */
7531         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7532                 struct thread *t = &p->threads[i];
7533                 struct table *table;
7534
7535                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7536                 CHECK(t->tables, ENOMEM);
7537
7538                 TAILQ_FOREACH(table, &p->tables, node) {
7539                         struct table_runtime *r = &t->tables[table->id];
7540
7541                         if (table->type) {
7542                                 uint64_t size;
7543
7544                                 size = table->type->ops.mailbox_size_get();
7545
7546                                 /* r->func. */
7547                                 r->func = table->type->ops.lkp;
7548
7549                                 /* r->mailbox. */
7550                                 if (size) {
7551                                         r->mailbox = calloc(1, size);
7552                                         CHECK(r->mailbox, ENOMEM);
7553                                 }
7554
7555                                 /* r->key. */
7556                                 r->key = table->header ?
7557                                         &t->structs[table->header->struct_id] :
7558                                         &t->structs[p->metadata_struct_id];
7559                         } else {
7560                                 r->func = table_stub_lkp;
7561                         }
7562                 }
7563         }
7564
7565         return 0;
7566 }
7567
7568 static void
7569 table_build_free(struct rte_swx_pipeline *p)
7570 {
7571         uint32_t i;
7572
7573         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7574                 struct thread *t = &p->threads[i];
7575                 uint32_t j;
7576
7577                 if (!t->tables)
7578                         continue;
7579
7580                 for (j = 0; j < p->n_tables; j++) {
7581                         struct table_runtime *r = &t->tables[j];
7582
7583                         free(r->mailbox);
7584                 }
7585
7586                 free(t->tables);
7587                 t->tables = NULL;
7588         }
7589
7590         if (p->table_stats) {
7591                 for (i = 0; i < p->n_tables; i++)
7592                         free(p->table_stats[i].n_pkts_action);
7593
7594                 free(p->table_stats);
7595         }
7596 }
7597
7598 static void
7599 table_free(struct rte_swx_pipeline *p)
7600 {
7601         table_build_free(p);
7602
7603         /* Tables. */
7604         for ( ; ; ) {
7605                 struct table *elem;
7606
7607                 elem = TAILQ_FIRST(&p->tables);
7608                 if (!elem)
7609                         break;
7610
7611                 TAILQ_REMOVE(&p->tables, elem, node);
7612                 free(elem->fields);
7613                 free(elem->actions);
7614                 free(elem->default_action_data);
7615                 free(elem);
7616         }
7617
7618         /* Table types. */
7619         for ( ; ; ) {
7620                 struct table_type *elem;
7621
7622                 elem = TAILQ_FIRST(&p->table_types);
7623                 if (!elem)
7624                         break;
7625
7626                 TAILQ_REMOVE(&p->table_types, elem, node);
7627                 free(elem);
7628         }
7629 }
7630
7631 /*
7632  * Selector.
7633  */
7634 static struct selector *
7635 selector_find(struct rte_swx_pipeline *p, const char *name)
7636 {
7637         struct selector *s;
7638
7639         TAILQ_FOREACH(s, &p->selectors, node)
7640                 if (strcmp(s->name, name) == 0)
7641                         return s;
7642
7643         return NULL;
7644 }
7645
7646 static struct selector *
7647 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7648 {
7649         struct selector *s = NULL;
7650
7651         TAILQ_FOREACH(s, &p->selectors, node)
7652                 if (s->id == id)
7653                         return s;
7654
7655         return NULL;
7656 }
7657
7658 static int
7659 selector_fields_check(struct rte_swx_pipeline *p,
7660                       struct rte_swx_pipeline_selector_params *params,
7661                       struct header **header)
7662 {
7663         struct header *h0 = NULL;
7664         struct field *hf, *mf;
7665         uint32_t i;
7666
7667         /* Return if no selector fields. */
7668         if (!params->n_selector_fields || !params->selector_field_names)
7669                 return -EINVAL;
7670
7671         /* Check that all the selector fields either belong to the same header
7672          * or are all meta-data fields.
7673          */
7674         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7675         mf = metadata_field_parse(p, params->selector_field_names[0]);
7676         if (!hf && !mf)
7677                 return -EINVAL;
7678
7679         for (i = 1; i < params->n_selector_fields; i++)
7680                 if (h0) {
7681                         struct header *h;
7682
7683                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7684                         if (!hf || (h->id != h0->id))
7685                                 return -EINVAL;
7686                 } else {
7687                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7688                         if (!mf)
7689                                 return -EINVAL;
7690                 }
7691
7692         /* Check that there are no duplicated match fields. */
7693         for (i = 0; i < params->n_selector_fields; i++) {
7694                 const char *field_name = params->selector_field_names[i];
7695                 uint32_t j;
7696
7697                 for (j = i + 1; j < params->n_selector_fields; j++)
7698                         if (!strcmp(params->selector_field_names[j], field_name))
7699                                 return -EINVAL;
7700         }
7701
7702         /* Return. */
7703         if (header)
7704                 *header = h0;
7705
7706         return 0;
7707 }
7708
7709 int
7710 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7711                                  const char *name,
7712                                  struct rte_swx_pipeline_selector_params *params)
7713 {
7714         struct selector *s;
7715         struct header *selector_header = NULL;
7716         struct field *group_id_field, *member_id_field;
7717         uint32_t i;
7718         int status = 0;
7719
7720         CHECK(p, EINVAL);
7721
7722         CHECK_NAME(name, EINVAL);
7723         CHECK(!table_find(p, name), EEXIST);
7724         CHECK(!selector_find(p, name), EEXIST);
7725         CHECK(!learner_find(p, name), EEXIST);
7726
7727         CHECK(params, EINVAL);
7728
7729         CHECK_NAME(params->group_id_field_name, EINVAL);
7730         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7731         CHECK(group_id_field, EINVAL);
7732
7733         for (i = 0; i < params->n_selector_fields; i++) {
7734                 const char *field_name = params->selector_field_names[i];
7735
7736                 CHECK_NAME(field_name, EINVAL);
7737         }
7738         status = selector_fields_check(p, params, &selector_header);
7739         if (status)
7740                 return status;
7741
7742         CHECK_NAME(params->member_id_field_name, EINVAL);
7743         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7744         CHECK(member_id_field, EINVAL);
7745
7746         CHECK(params->n_groups_max, EINVAL);
7747
7748         CHECK(params->n_members_per_group_max, EINVAL);
7749
7750         /* Memory allocation. */
7751         s = calloc(1, sizeof(struct selector));
7752         if (!s) {
7753                 status = -ENOMEM;
7754                 goto error;
7755         }
7756
7757         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7758         if (!s->selector_fields) {
7759                 status = -ENOMEM;
7760                 goto error;
7761         }
7762
7763         /* Node initialization. */
7764         strcpy(s->name, name);
7765
7766         s->group_id_field = group_id_field;
7767
7768         for (i = 0; i < params->n_selector_fields; i++) {
7769                 const char *field_name = params->selector_field_names[i];
7770
7771                 s->selector_fields[i] = selector_header ?
7772                         header_field_parse(p, field_name, NULL) :
7773                         metadata_field_parse(p, field_name);
7774         }
7775
7776         s->n_selector_fields = params->n_selector_fields;
7777
7778         s->selector_header = selector_header;
7779
7780         s->member_id_field = member_id_field;
7781
7782         s->n_groups_max = params->n_groups_max;
7783
7784         s->n_members_per_group_max = params->n_members_per_group_max;
7785
7786         s->id = p->n_selectors;
7787
7788         /* Node add to tailq. */
7789         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7790         p->n_selectors++;
7791
7792         return 0;
7793
7794 error:
7795         if (!s)
7796                 return status;
7797
7798         free(s->selector_fields);
7799
7800         free(s);
7801
7802         return status;
7803 }
7804
7805 static void
7806 selector_params_free(struct rte_swx_table_selector_params *params)
7807 {
7808         if (!params)
7809                 return;
7810
7811         free(params->selector_mask);
7812
7813         free(params);
7814 }
7815
7816 static struct rte_swx_table_selector_params *
7817 selector_table_params_get(struct selector *s)
7818 {
7819         struct rte_swx_table_selector_params *params = NULL;
7820         struct field *first, *last;
7821         uint32_t i;
7822
7823         /* Memory allocation. */
7824         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7825         if (!params)
7826                 goto error;
7827
7828         /* Group ID. */
7829         params->group_id_offset = s->group_id_field->offset / 8;
7830
7831         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7832         first = s->selector_fields[0];
7833         last = s->selector_fields[0];
7834
7835         for (i = 0; i < s->n_selector_fields; i++) {
7836                 struct field *f = s->selector_fields[i];
7837
7838                 if (f->offset < first->offset)
7839                         first = f;
7840
7841                 if (f->offset > last->offset)
7842                         last = f;
7843         }
7844
7845         /* Selector offset and size. */
7846         params->selector_offset = first->offset / 8;
7847         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7848
7849         /* Memory allocation. */
7850         params->selector_mask = calloc(1, params->selector_size);
7851         if (!params->selector_mask)
7852                 goto error;
7853
7854         /* Selector mask. */
7855         for (i = 0; i < s->n_selector_fields; i++) {
7856                 struct field *f = s->selector_fields[i];
7857                 uint32_t start = (f->offset - first->offset) / 8;
7858                 size_t size = f->n_bits / 8;
7859
7860                 memset(&params->selector_mask[start], 0xFF, size);
7861         }
7862
7863         /* Member ID. */
7864         params->member_id_offset = s->member_id_field->offset / 8;
7865
7866         /* Maximum number of groups. */
7867         params->n_groups_max = s->n_groups_max;
7868
7869         /* Maximum number of members per group. */
7870         params->n_members_per_group_max = s->n_members_per_group_max;
7871
7872         return params;
7873
7874 error:
7875         selector_params_free(params);
7876         return NULL;
7877 }
7878
7879 static void
7880 selector_build_free(struct rte_swx_pipeline *p)
7881 {
7882         uint32_t i;
7883
7884         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7885                 struct thread *t = &p->threads[i];
7886                 uint32_t j;
7887
7888                 if (!t->selectors)
7889                         continue;
7890
7891                 for (j = 0; j < p->n_selectors; j++) {
7892                         struct selector_runtime *r = &t->selectors[j];
7893
7894                         free(r->mailbox);
7895                 }
7896
7897                 free(t->selectors);
7898                 t->selectors = NULL;
7899         }
7900
7901         free(p->selector_stats);
7902         p->selector_stats = NULL;
7903 }
7904
7905 static int
7906 selector_build(struct rte_swx_pipeline *p)
7907 {
7908         uint32_t i;
7909         int status = 0;
7910
7911         /* Per pipeline: selector statistics. */
7912         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7913         if (!p->selector_stats) {
7914                 status = -ENOMEM;
7915                 goto error;
7916         }
7917
7918         /* Per thread: selector run-time. */
7919         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7920                 struct thread *t = &p->threads[i];
7921                 struct selector *s;
7922
7923                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7924                 if (!t->selectors) {
7925                         status = -ENOMEM;
7926                         goto error;
7927                 }
7928
7929                 TAILQ_FOREACH(s, &p->selectors, node) {
7930                         struct selector_runtime *r = &t->selectors[s->id];
7931                         uint64_t size;
7932
7933                         /* r->mailbox. */
7934                         size = rte_swx_table_selector_mailbox_size_get();
7935                         if (size) {
7936                                 r->mailbox = calloc(1, size);
7937                                 if (!r->mailbox) {
7938                                         status = -ENOMEM;
7939                                         goto error;
7940                                 }
7941                         }
7942
7943                         /* r->group_id_buffer. */
7944                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
7945
7946                         /* r->selector_buffer. */
7947                         r->selector_buffer = s->selector_header ?
7948                                 &t->structs[s->selector_header->struct_id] :
7949                                 &t->structs[p->metadata_struct_id];
7950
7951                         /* r->member_id_buffer. */
7952                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
7953                 }
7954         }
7955
7956         return 0;
7957
7958 error:
7959         selector_build_free(p);
7960         return status;
7961 }
7962
7963 static void
7964 selector_free(struct rte_swx_pipeline *p)
7965 {
7966         selector_build_free(p);
7967
7968         /* Selector tables. */
7969         for ( ; ; ) {
7970                 struct selector *elem;
7971
7972                 elem = TAILQ_FIRST(&p->selectors);
7973                 if (!elem)
7974                         break;
7975
7976                 TAILQ_REMOVE(&p->selectors, elem, node);
7977                 free(elem->selector_fields);
7978                 free(elem);
7979         }
7980 }
7981
7982 /*
7983  * Learner table.
7984  */
7985 static struct learner *
7986 learner_find(struct rte_swx_pipeline *p, const char *name)
7987 {
7988         struct learner *l;
7989
7990         TAILQ_FOREACH(l, &p->learners, node)
7991                 if (!strcmp(l->name, name))
7992                         return l;
7993
7994         return NULL;
7995 }
7996
7997 static struct learner *
7998 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7999 {
8000         struct learner *l = NULL;
8001
8002         TAILQ_FOREACH(l, &p->learners, node)
8003                 if (l->id == id)
8004                         return l;
8005
8006         return NULL;
8007 }
8008
8009 static int
8010 learner_match_fields_check(struct rte_swx_pipeline *p,
8011                            struct rte_swx_pipeline_learner_params *params,
8012                            struct header **header)
8013 {
8014         struct header *h0 = NULL;
8015         struct field *hf, *mf;
8016         uint32_t i;
8017
8018         /* Return if no match fields. */
8019         if (!params->n_fields || !params->field_names)
8020                 return -EINVAL;
8021
8022         /* Check that all the match fields either belong to the same header
8023          * or are all meta-data fields.
8024          */
8025         hf = header_field_parse(p, params->field_names[0], &h0);
8026         mf = metadata_field_parse(p, params->field_names[0]);
8027         if (!hf && !mf)
8028                 return -EINVAL;
8029
8030         for (i = 1; i < params->n_fields; i++)
8031                 if (h0) {
8032                         struct header *h;
8033
8034                         hf = header_field_parse(p, params->field_names[i], &h);
8035                         if (!hf || (h->id != h0->id))
8036                                 return -EINVAL;
8037                 } else {
8038                         mf = metadata_field_parse(p, params->field_names[i]);
8039                         if (!mf)
8040                                 return -EINVAL;
8041                 }
8042
8043         /* Check that there are no duplicated match fields. */
8044         for (i = 0; i < params->n_fields; i++) {
8045                 const char *field_name = params->field_names[i];
8046                 uint32_t j;
8047
8048                 for (j = i + 1; j < params->n_fields; j++)
8049                         if (!strcmp(params->field_names[j], field_name))
8050                                 return -EINVAL;
8051         }
8052
8053         /* Return. */
8054         if (header)
8055                 *header = h0;
8056
8057         return 0;
8058 }
8059
8060 static int
8061 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8062 {
8063         struct struct_type *mst = p->metadata_st, *ast = a->st;
8064         struct field *mf, *af;
8065         uint32_t mf_pos, i;
8066
8067         if (!ast) {
8068                 if (mf_name)
8069                         return -EINVAL;
8070
8071                 return 0;
8072         }
8073
8074         /* Check that mf_name is the name of a valid meta-data field. */
8075         CHECK_NAME(mf_name, EINVAL);
8076         mf = metadata_field_parse(p, mf_name);
8077         CHECK(mf, EINVAL);
8078
8079         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8080          * all the action arguments.
8081          */
8082         mf_pos = mf - mst->fields;
8083         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8084
8085         /* Check that the size of each of the identified meta-data fields matches exactly the size
8086          * of the corresponding action argument.
8087          */
8088         for (i = 0; i < ast->n_fields; i++) {
8089                 mf = &mst->fields[mf_pos + i];
8090                 af = &ast->fields[i];
8091
8092                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8093         }
8094
8095         return 0;
8096 }
8097
8098 static int
8099 learner_action_learning_check(struct rte_swx_pipeline *p,
8100                               struct action *action,
8101                               const char **action_names,
8102                               uint32_t n_actions)
8103 {
8104         uint32_t i;
8105
8106         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8107          * the action passed as argument to the "learn" instruction) is also enabled for the
8108          * current learner table.
8109          */
8110         for (i = 0; i < action->n_instructions; i++) {
8111                 struct instruction *instr = &action->instructions[i];
8112                 uint32_t found = 0, j;
8113
8114                 if (instr->type != INSTR_LEARNER_LEARN)
8115                         continue;
8116
8117                 for (j = 0; j < n_actions; j++) {
8118                         struct action *a;
8119
8120                         a = action_find(p, action_names[j]);
8121                         if (!a)
8122                                 return -EINVAL;
8123
8124                         if (a->id == instr->learn.action_id)
8125                                 found = 1;
8126                 }
8127
8128                 if (!found)
8129                         return -EINVAL;
8130         }
8131
8132         return 0;
8133 }
8134
8135 int
8136 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8137                               const char *name,
8138                               struct rte_swx_pipeline_learner_params *params,
8139                               uint32_t size,
8140                               uint32_t timeout)
8141 {
8142         struct learner *l = NULL;
8143         struct action *default_action;
8144         struct header *header = NULL;
8145         uint32_t action_data_size_max = 0, i;
8146         int status = 0;
8147
8148         CHECK(p, EINVAL);
8149
8150         CHECK_NAME(name, EINVAL);
8151         CHECK(!table_find(p, name), EEXIST);
8152         CHECK(!selector_find(p, name), EEXIST);
8153         CHECK(!learner_find(p, name), EEXIST);
8154
8155         CHECK(params, EINVAL);
8156
8157         /* Match checks. */
8158         status = learner_match_fields_check(p, params, &header);
8159         if (status)
8160                 return status;
8161
8162         /* Action checks. */
8163         CHECK(params->n_actions, EINVAL);
8164
8165         CHECK(params->action_names, EINVAL);
8166         for (i = 0; i < params->n_actions; i++) {
8167                 const char *action_name = params->action_names[i];
8168                 const char *action_field_name = params->action_field_names[i];
8169                 struct action *a;
8170                 uint32_t action_data_size;
8171
8172                 CHECK_NAME(action_name, EINVAL);
8173
8174                 a = action_find(p, action_name);
8175                 CHECK(a, EINVAL);
8176
8177                 status = learner_action_args_check(p, a, action_field_name);
8178                 if (status)
8179                         return status;
8180
8181                 status = learner_action_learning_check(p,
8182                                                        a,
8183                                                        params->action_names,
8184                                                        params->n_actions);
8185                 if (status)
8186                         return status;
8187
8188                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8189                 if (action_data_size > action_data_size_max)
8190                         action_data_size_max = action_data_size;
8191         }
8192
8193         CHECK_NAME(params->default_action_name, EINVAL);
8194         for (i = 0; i < p->n_actions; i++)
8195                 if (!strcmp(params->action_names[i],
8196                             params->default_action_name))
8197                         break;
8198         CHECK(i < params->n_actions, EINVAL);
8199
8200         default_action = action_find(p, params->default_action_name);
8201         CHECK((default_action->st && params->default_action_data) ||
8202               !params->default_action_data, EINVAL);
8203
8204         /* Any other checks. */
8205         CHECK(size, EINVAL);
8206         CHECK(timeout, EINVAL);
8207
8208         /* Memory allocation. */
8209         l = calloc(1, sizeof(struct learner));
8210         if (!l)
8211                 goto nomem;
8212
8213         l->fields = calloc(params->n_fields, sizeof(struct field *));
8214         if (!l->fields)
8215                 goto nomem;
8216
8217         l->actions = calloc(params->n_actions, sizeof(struct action *));
8218         if (!l->actions)
8219                 goto nomem;
8220
8221         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
8222         if (!l->action_arg)
8223                 goto nomem;
8224
8225         if (action_data_size_max) {
8226                 l->default_action_data = calloc(1, action_data_size_max);
8227                 if (!l->default_action_data)
8228                         goto nomem;
8229         }
8230
8231         /* Node initialization. */
8232         strcpy(l->name, name);
8233
8234         for (i = 0; i < params->n_fields; i++) {
8235                 const char *field_name = params->field_names[i];
8236
8237                 l->fields[i] = header ?
8238                         header_field_parse(p, field_name, NULL) :
8239                         metadata_field_parse(p, field_name);
8240         }
8241
8242         l->n_fields = params->n_fields;
8243
8244         l->header = header;
8245
8246         for (i = 0; i < params->n_actions; i++) {
8247                 const char *mf_name = params->action_field_names[i];
8248
8249                 l->actions[i] = action_find(p, params->action_names[i]);
8250
8251                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
8252         }
8253
8254         l->default_action = default_action;
8255
8256         if (default_action->st)
8257                 memcpy(l->default_action_data,
8258                        params->default_action_data,
8259                        default_action->st->n_bits / 8);
8260
8261         l->n_actions = params->n_actions;
8262
8263         l->default_action_is_const = params->default_action_is_const;
8264
8265         l->action_data_size_max = action_data_size_max;
8266
8267         l->size = size;
8268
8269         l->timeout = timeout;
8270
8271         l->id = p->n_learners;
8272
8273         /* Node add to tailq. */
8274         TAILQ_INSERT_TAIL(&p->learners, l, node);
8275         p->n_learners++;
8276
8277         return 0;
8278
8279 nomem:
8280         if (!l)
8281                 return -ENOMEM;
8282
8283         free(l->action_arg);
8284         free(l->actions);
8285         free(l->fields);
8286         free(l);
8287
8288         return -ENOMEM;
8289 }
8290
8291 static void
8292 learner_params_free(struct rte_swx_table_learner_params *params)
8293 {
8294         if (!params)
8295                 return;
8296
8297         free(params->key_mask0);
8298
8299         free(params);
8300 }
8301
8302 static struct rte_swx_table_learner_params *
8303 learner_params_get(struct learner *l)
8304 {
8305         struct rte_swx_table_learner_params *params = NULL;
8306         struct field *first, *last;
8307         uint32_t i;
8308
8309         /* Memory allocation. */
8310         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8311         if (!params)
8312                 goto error;
8313
8314         /* Find first (smallest offset) and last (biggest offset) match fields. */
8315         first = l->fields[0];
8316         last = l->fields[0];
8317
8318         for (i = 0; i < l->n_fields; i++) {
8319                 struct field *f = l->fields[i];
8320
8321                 if (f->offset < first->offset)
8322                         first = f;
8323
8324                 if (f->offset > last->offset)
8325                         last = f;
8326         }
8327
8328         /* Key offset and size. */
8329         params->key_offset = first->offset / 8;
8330         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8331
8332         /* Memory allocation. */
8333         params->key_mask0 = calloc(1, params->key_size);
8334         if (!params->key_mask0)
8335                 goto error;
8336
8337         /* Key mask. */
8338         for (i = 0; i < l->n_fields; i++) {
8339                 struct field *f = l->fields[i];
8340                 uint32_t start = (f->offset - first->offset) / 8;
8341                 size_t size = f->n_bits / 8;
8342
8343                 memset(&params->key_mask0[start], 0xFF, size);
8344         }
8345
8346         /* Action data size. */
8347         params->action_data_size = l->action_data_size_max;
8348
8349         /* Maximum number of keys. */
8350         params->n_keys_max = l->size;
8351
8352         /* Timeout. */
8353         params->key_timeout = l->timeout;
8354
8355         return params;
8356
8357 error:
8358         learner_params_free(params);
8359         return NULL;
8360 }
8361
8362 static void
8363 learner_build_free(struct rte_swx_pipeline *p)
8364 {
8365         uint32_t i;
8366
8367         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8368                 struct thread *t = &p->threads[i];
8369                 uint32_t j;
8370
8371                 if (!t->learners)
8372                         continue;
8373
8374                 for (j = 0; j < p->n_learners; j++) {
8375                         struct learner_runtime *r = &t->learners[j];
8376
8377                         free(r->mailbox);
8378                         free(r->action_data);
8379                 }
8380
8381                 free(t->learners);
8382                 t->learners = NULL;
8383         }
8384
8385         if (p->learner_stats) {
8386                 for (i = 0; i < p->n_learners; i++)
8387                         free(p->learner_stats[i].n_pkts_action);
8388
8389                 free(p->learner_stats);
8390         }
8391 }
8392
8393 static int
8394 learner_build(struct rte_swx_pipeline *p)
8395 {
8396         uint32_t i;
8397         int status = 0;
8398
8399         /* Per pipeline: learner statistics. */
8400         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8401         CHECK(p->learner_stats, ENOMEM);
8402
8403         for (i = 0; i < p->n_learners; i++) {
8404                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8405                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8406         }
8407
8408         /* Per thread: learner run-time. */
8409         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8410                 struct thread *t = &p->threads[i];
8411                 struct learner *l;
8412
8413                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8414                 if (!t->learners) {
8415                         status = -ENOMEM;
8416                         goto error;
8417                 }
8418
8419                 TAILQ_FOREACH(l, &p->learners, node) {
8420                         struct learner_runtime *r = &t->learners[l->id];
8421                         uint64_t size;
8422                         uint32_t j;
8423
8424                         /* r->mailbox. */
8425                         size = rte_swx_table_learner_mailbox_size_get();
8426                         if (size) {
8427                                 r->mailbox = calloc(1, size);
8428                                 if (!r->mailbox) {
8429                                         status = -ENOMEM;
8430                                         goto error;
8431                                 }
8432                         }
8433
8434                         /* r->key. */
8435                         r->key = l->header ?
8436                                 &t->structs[l->header->struct_id] :
8437                                 &t->structs[p->metadata_struct_id];
8438
8439                         /* r->action_data. */
8440                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
8441                         if (!r->action_data) {
8442                                 status = -ENOMEM;
8443                                 goto error;
8444                         }
8445
8446                         for (j = 0; j < l->n_actions; j++) {
8447                                 struct action *a = l->actions[j];
8448                                 struct field *mf = l->action_arg[j];
8449                                 uint8_t *m = t->structs[p->metadata_struct_id];
8450
8451                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
8452                         }
8453                 }
8454         }
8455
8456         return 0;
8457
8458 error:
8459         learner_build_free(p);
8460         return status;
8461 }
8462
8463 static void
8464 learner_free(struct rte_swx_pipeline *p)
8465 {
8466         learner_build_free(p);
8467
8468         /* Learner tables. */
8469         for ( ; ; ) {
8470                 struct learner *l;
8471
8472                 l = TAILQ_FIRST(&p->learners);
8473                 if (!l)
8474                         break;
8475
8476                 TAILQ_REMOVE(&p->learners, l, node);
8477                 free(l->fields);
8478                 free(l->actions);
8479                 free(l->action_arg);
8480                 free(l->default_action_data);
8481                 free(l);
8482         }
8483 }
8484
8485 /*
8486  * Table state.
8487  */
8488 static int
8489 table_state_build(struct rte_swx_pipeline *p)
8490 {
8491         struct table *table;
8492         struct selector *s;
8493         struct learner *l;
8494
8495         p->table_state = calloc(p->n_tables + p->n_selectors,
8496                                 sizeof(struct rte_swx_table_state));
8497         CHECK(p->table_state, ENOMEM);
8498
8499         TAILQ_FOREACH(table, &p->tables, node) {
8500                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8501
8502                 if (table->type) {
8503                         struct rte_swx_table_params *params;
8504
8505                         /* ts->obj. */
8506                         params = table_params_get(table);
8507                         CHECK(params, ENOMEM);
8508
8509                         ts->obj = table->type->ops.create(params,
8510                                 NULL,
8511                                 table->args,
8512                                 p->numa_node);
8513
8514                         table_params_free(params);
8515                         CHECK(ts->obj, ENODEV);
8516                 }
8517
8518                 /* ts->default_action_data. */
8519                 if (table->action_data_size_max) {
8520                         ts->default_action_data =
8521                                 malloc(table->action_data_size_max);
8522                         CHECK(ts->default_action_data, ENOMEM);
8523
8524                         memcpy(ts->default_action_data,
8525                                table->default_action_data,
8526                                table->action_data_size_max);
8527                 }
8528
8529                 /* ts->default_action_id. */
8530                 ts->default_action_id = table->default_action->id;
8531         }
8532
8533         TAILQ_FOREACH(s, &p->selectors, node) {
8534                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8535                 struct rte_swx_table_selector_params *params;
8536
8537                 /* ts->obj. */
8538                 params = selector_table_params_get(s);
8539                 CHECK(params, ENOMEM);
8540
8541                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8542
8543                 selector_params_free(params);
8544                 CHECK(ts->obj, ENODEV);
8545         }
8546
8547         TAILQ_FOREACH(l, &p->learners, node) {
8548                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8549                         p->n_selectors + l->id];
8550                 struct rte_swx_table_learner_params *params;
8551
8552                 /* ts->obj. */
8553                 params = learner_params_get(l);
8554                 CHECK(params, ENOMEM);
8555
8556                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8557                 learner_params_free(params);
8558                 CHECK(ts->obj, ENODEV);
8559
8560                 /* ts->default_action_data. */
8561                 if (l->action_data_size_max) {
8562                         ts->default_action_data = malloc(l->action_data_size_max);
8563                         CHECK(ts->default_action_data, ENOMEM);
8564
8565                         memcpy(ts->default_action_data,
8566                                l->default_action_data,
8567                                l->action_data_size_max);
8568                 }
8569
8570                 /* ts->default_action_id. */
8571                 ts->default_action_id = l->default_action->id;
8572         }
8573
8574         return 0;
8575 }
8576
8577 static void
8578 table_state_build_free(struct rte_swx_pipeline *p)
8579 {
8580         uint32_t i;
8581
8582         if (!p->table_state)
8583                 return;
8584
8585         for (i = 0; i < p->n_tables; i++) {
8586                 struct rte_swx_table_state *ts = &p->table_state[i];
8587                 struct table *table = table_find_by_id(p, i);
8588
8589                 /* ts->obj. */
8590                 if (table->type && ts->obj)
8591                         table->type->ops.free(ts->obj);
8592
8593                 /* ts->default_action_data. */
8594                 free(ts->default_action_data);
8595         }
8596
8597         for (i = 0; i < p->n_selectors; i++) {
8598                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8599
8600                 /* ts->obj. */
8601                 if (ts->obj)
8602                         rte_swx_table_selector_free(ts->obj);
8603         }
8604
8605         for (i = 0; i < p->n_learners; i++) {
8606                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8607
8608                 /* ts->obj. */
8609                 if (ts->obj)
8610                         rte_swx_table_learner_free(ts->obj);
8611
8612                 /* ts->default_action_data. */
8613                 free(ts->default_action_data);
8614         }
8615
8616         free(p->table_state);
8617         p->table_state = NULL;
8618 }
8619
8620 static void
8621 table_state_free(struct rte_swx_pipeline *p)
8622 {
8623         table_state_build_free(p);
8624 }
8625
8626 /*
8627  * Register array.
8628  */
8629 static struct regarray *
8630 regarray_find(struct rte_swx_pipeline *p, const char *name)
8631 {
8632         struct regarray *elem;
8633
8634         TAILQ_FOREACH(elem, &p->regarrays, node)
8635                 if (!strcmp(elem->name, name))
8636                         return elem;
8637
8638         return NULL;
8639 }
8640
8641 static struct regarray *
8642 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8643 {
8644         struct regarray *elem = NULL;
8645
8646         TAILQ_FOREACH(elem, &p->regarrays, node)
8647                 if (elem->id == id)
8648                         return elem;
8649
8650         return NULL;
8651 }
8652
8653 int
8654 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8655                               const char *name,
8656                               uint32_t size,
8657                               uint64_t init_val)
8658 {
8659         struct regarray *r;
8660
8661         CHECK(p, EINVAL);
8662
8663         CHECK_NAME(name, EINVAL);
8664         CHECK(!regarray_find(p, name), EEXIST);
8665
8666         CHECK(size, EINVAL);
8667         size = rte_align32pow2(size);
8668
8669         /* Memory allocation. */
8670         r = calloc(1, sizeof(struct regarray));
8671         CHECK(r, ENOMEM);
8672
8673         /* Node initialization. */
8674         strcpy(r->name, name);
8675         r->init_val = init_val;
8676         r->size = size;
8677         r->id = p->n_regarrays;
8678
8679         /* Node add to tailq. */
8680         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8681         p->n_regarrays++;
8682
8683         return 0;
8684 }
8685
8686 static int
8687 regarray_build(struct rte_swx_pipeline *p)
8688 {
8689         struct regarray *regarray;
8690
8691         if (!p->n_regarrays)
8692                 return 0;
8693
8694         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8695         CHECK(p->regarray_runtime, ENOMEM);
8696
8697         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8698                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8699                 uint32_t i;
8700
8701                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8702                                          RTE_CACHE_LINE_SIZE,
8703                                          p->numa_node);
8704                 CHECK(r->regarray, ENOMEM);
8705
8706                 if (regarray->init_val)
8707                         for (i = 0; i < regarray->size; i++)
8708                                 r->regarray[i] = regarray->init_val;
8709
8710                 r->size_mask = regarray->size - 1;
8711         }
8712
8713         return 0;
8714 }
8715
8716 static void
8717 regarray_build_free(struct rte_swx_pipeline *p)
8718 {
8719         uint32_t i;
8720
8721         if (!p->regarray_runtime)
8722                 return;
8723
8724         for (i = 0; i < p->n_regarrays; i++) {
8725                 struct regarray *regarray = regarray_find_by_id(p, i);
8726                 struct regarray_runtime *r = &p->regarray_runtime[i];
8727
8728                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8729         }
8730
8731         free(p->regarray_runtime);
8732         p->regarray_runtime = NULL;
8733 }
8734
8735 static void
8736 regarray_free(struct rte_swx_pipeline *p)
8737 {
8738         regarray_build_free(p);
8739
8740         for ( ; ; ) {
8741                 struct regarray *elem;
8742
8743                 elem = TAILQ_FIRST(&p->regarrays);
8744                 if (!elem)
8745                         break;
8746
8747                 TAILQ_REMOVE(&p->regarrays, elem, node);
8748                 free(elem);
8749         }
8750 }
8751
8752 /*
8753  * Meter array.
8754  */
8755 static struct meter_profile *
8756 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8757 {
8758         struct meter_profile *elem;
8759
8760         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8761                 if (!strcmp(elem->name, name))
8762                         return elem;
8763
8764         return NULL;
8765 }
8766
8767 static struct metarray *
8768 metarray_find(struct rte_swx_pipeline *p, const char *name)
8769 {
8770         struct metarray *elem;
8771
8772         TAILQ_FOREACH(elem, &p->metarrays, node)
8773                 if (!strcmp(elem->name, name))
8774                         return elem;
8775
8776         return NULL;
8777 }
8778
8779 static struct metarray *
8780 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8781 {
8782         struct metarray *elem = NULL;
8783
8784         TAILQ_FOREACH(elem, &p->metarrays, node)
8785                 if (elem->id == id)
8786                         return elem;
8787
8788         return NULL;
8789 }
8790
8791 int
8792 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8793                                  const char *name,
8794                                  uint32_t size)
8795 {
8796         struct metarray *m;
8797
8798         CHECK(p, EINVAL);
8799
8800         CHECK_NAME(name, EINVAL);
8801         CHECK(!metarray_find(p, name), EEXIST);
8802
8803         CHECK(size, EINVAL);
8804         size = rte_align32pow2(size);
8805
8806         /* Memory allocation. */
8807         m = calloc(1, sizeof(struct metarray));
8808         CHECK(m, ENOMEM);
8809
8810         /* Node initialization. */
8811         strcpy(m->name, name);
8812         m->size = size;
8813         m->id = p->n_metarrays;
8814
8815         /* Node add to tailq. */
8816         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8817         p->n_metarrays++;
8818
8819         return 0;
8820 }
8821
8822 struct meter_profile meter_profile_default = {
8823         .node = {0},
8824         .name = "",
8825         .params = {0},
8826
8827         .profile = {
8828                 .cbs = 10000,
8829                 .pbs = 10000,
8830                 .cir_period = 1,
8831                 .cir_bytes_per_period = 1,
8832                 .pir_period = 1,
8833                 .pir_bytes_per_period = 1,
8834         },
8835
8836         .n_users = 0,
8837 };
8838
8839 static void
8840 meter_init(struct meter *m)
8841 {
8842         memset(m, 0, sizeof(struct meter));
8843         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8844         m->profile = &meter_profile_default;
8845         m->color_mask = RTE_COLOR_GREEN;
8846
8847         meter_profile_default.n_users++;
8848 }
8849
8850 static int
8851 metarray_build(struct rte_swx_pipeline *p)
8852 {
8853         struct metarray *m;
8854
8855         if (!p->n_metarrays)
8856                 return 0;
8857
8858         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8859         CHECK(p->metarray_runtime, ENOMEM);
8860
8861         TAILQ_FOREACH(m, &p->metarrays, node) {
8862                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8863                 uint32_t i;
8864
8865                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8866                                          RTE_CACHE_LINE_SIZE,
8867                                          p->numa_node);
8868                 CHECK(r->metarray, ENOMEM);
8869
8870                 for (i = 0; i < m->size; i++)
8871                         meter_init(&r->metarray[i]);
8872
8873                 r->size_mask = m->size - 1;
8874         }
8875
8876         return 0;
8877 }
8878
8879 static void
8880 metarray_build_free(struct rte_swx_pipeline *p)
8881 {
8882         uint32_t i;
8883
8884         if (!p->metarray_runtime)
8885                 return;
8886
8887         for (i = 0; i < p->n_metarrays; i++) {
8888                 struct metarray *m = metarray_find_by_id(p, i);
8889                 struct metarray_runtime *r = &p->metarray_runtime[i];
8890
8891                 env_free(r->metarray, m->size * sizeof(struct meter));
8892         }
8893
8894         free(p->metarray_runtime);
8895         p->metarray_runtime = NULL;
8896 }
8897
8898 static void
8899 metarray_free(struct rte_swx_pipeline *p)
8900 {
8901         metarray_build_free(p);
8902
8903         /* Meter arrays. */
8904         for ( ; ; ) {
8905                 struct metarray *elem;
8906
8907                 elem = TAILQ_FIRST(&p->metarrays);
8908                 if (!elem)
8909                         break;
8910
8911                 TAILQ_REMOVE(&p->metarrays, elem, node);
8912                 free(elem);
8913         }
8914
8915         /* Meter profiles. */
8916         for ( ; ; ) {
8917                 struct meter_profile *elem;
8918
8919                 elem = TAILQ_FIRST(&p->meter_profiles);
8920                 if (!elem)
8921                         break;
8922
8923                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8924                 free(elem);
8925         }
8926 }
8927
8928 /*
8929  * Pipeline.
8930  */
8931 int
8932 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8933 {
8934         struct rte_swx_pipeline *pipeline;
8935
8936         /* Check input parameters. */
8937         CHECK(p, EINVAL);
8938
8939         /* Memory allocation. */
8940         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8941         CHECK(pipeline, ENOMEM);
8942
8943         /* Initialization. */
8944         TAILQ_INIT(&pipeline->struct_types);
8945         TAILQ_INIT(&pipeline->port_in_types);
8946         TAILQ_INIT(&pipeline->ports_in);
8947         TAILQ_INIT(&pipeline->port_out_types);
8948         TAILQ_INIT(&pipeline->ports_out);
8949         TAILQ_INIT(&pipeline->extern_types);
8950         TAILQ_INIT(&pipeline->extern_objs);
8951         TAILQ_INIT(&pipeline->extern_funcs);
8952         TAILQ_INIT(&pipeline->headers);
8953         TAILQ_INIT(&pipeline->actions);
8954         TAILQ_INIT(&pipeline->table_types);
8955         TAILQ_INIT(&pipeline->tables);
8956         TAILQ_INIT(&pipeline->selectors);
8957         TAILQ_INIT(&pipeline->learners);
8958         TAILQ_INIT(&pipeline->regarrays);
8959         TAILQ_INIT(&pipeline->meter_profiles);
8960         TAILQ_INIT(&pipeline->metarrays);
8961
8962         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
8963         pipeline->numa_node = numa_node;
8964
8965         *p = pipeline;
8966         return 0;
8967 }
8968
8969 void
8970 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8971 {
8972         void *lib;
8973
8974         if (!p)
8975                 return;
8976
8977         lib = p->lib;
8978
8979         free(p->instruction_data);
8980         free(p->instructions);
8981
8982         metarray_free(p);
8983         regarray_free(p);
8984         table_state_free(p);
8985         learner_free(p);
8986         selector_free(p);
8987         table_free(p);
8988         action_free(p);
8989         instruction_table_free(p);
8990         metadata_free(p);
8991         header_free(p);
8992         extern_func_free(p);
8993         extern_obj_free(p);
8994         port_out_free(p);
8995         port_in_free(p);
8996         struct_free(p);
8997
8998         free(p);
8999
9000         if (lib)
9001                 dlclose(lib);
9002 }
9003
9004 int
9005 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9006                                      const char **instructions,
9007                                      uint32_t n_instructions)
9008 {
9009         int err;
9010         uint32_t i;
9011
9012         err = instruction_config(p, NULL, instructions, n_instructions);
9013         if (err)
9014                 return err;
9015
9016         /* Thread instruction pointer reset. */
9017         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9018                 struct thread *t = &p->threads[i];
9019
9020                 thread_ip_reset(p, t);
9021         }
9022
9023         return 0;
9024 }
9025
9026 static int
9027 pipeline_compile(struct rte_swx_pipeline *p);
9028
9029 int
9030 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9031 {
9032         int status;
9033
9034         CHECK(p, EINVAL);
9035         CHECK(p->build_done == 0, EEXIST);
9036
9037         status = port_in_build(p);
9038         if (status)
9039                 goto error;
9040
9041         status = port_out_build(p);
9042         if (status)
9043                 goto error;
9044
9045         status = struct_build(p);
9046         if (status)
9047                 goto error;
9048
9049         status = extern_obj_build(p);
9050         if (status)
9051                 goto error;
9052
9053         status = extern_func_build(p);
9054         if (status)
9055                 goto error;
9056
9057         status = header_build(p);
9058         if (status)
9059                 goto error;
9060
9061         status = metadata_build(p);
9062         if (status)
9063                 goto error;
9064
9065         status = instruction_table_build(p);
9066         if (status)
9067                 goto error;
9068
9069         status = action_build(p);
9070         if (status)
9071                 goto error;
9072
9073         status = table_build(p);
9074         if (status)
9075                 goto error;
9076
9077         status = selector_build(p);
9078         if (status)
9079                 goto error;
9080
9081         status = learner_build(p);
9082         if (status)
9083                 goto error;
9084
9085         status = table_state_build(p);
9086         if (status)
9087                 goto error;
9088
9089         status = regarray_build(p);
9090         if (status)
9091                 goto error;
9092
9093         status = metarray_build(p);
9094         if (status)
9095                 goto error;
9096
9097         p->build_done = 1;
9098
9099         pipeline_compile(p);
9100
9101         return 0;
9102
9103 error:
9104         metarray_build_free(p);
9105         regarray_build_free(p);
9106         table_state_build_free(p);
9107         learner_build_free(p);
9108         selector_build_free(p);
9109         table_build_free(p);
9110         action_build_free(p);
9111         instruction_table_build_free(p);
9112         metadata_build_free(p);
9113         header_build_free(p);
9114         extern_func_build_free(p);
9115         extern_obj_build_free(p);
9116         port_out_build_free(p);
9117         port_in_build_free(p);
9118         struct_build_free(p);
9119
9120         return status;
9121 }
9122
9123 void
9124 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9125 {
9126         uint32_t i;
9127
9128         for (i = 0; i < n_instructions; i++)
9129                 instr_exec(p);
9130 }
9131
9132 void
9133 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9134 {
9135         uint32_t i;
9136
9137         for (i = 0; i < p->n_ports_out; i++) {
9138                 struct port_out_runtime *port = &p->out[i];
9139
9140                 if (port->flush)
9141                         port->flush(port->obj);
9142         }
9143 }
9144
9145 /*
9146  * Control.
9147  */
9148 int
9149 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9150                               struct rte_swx_ctl_pipeline_info *pipeline)
9151 {
9152         struct action *action;
9153         struct table *table;
9154         uint32_t n_actions = 0, n_tables = 0;
9155
9156         if (!p || !pipeline)
9157                 return -EINVAL;
9158
9159         TAILQ_FOREACH(action, &p->actions, node)
9160                 n_actions++;
9161
9162         TAILQ_FOREACH(table, &p->tables, node)
9163                 n_tables++;
9164
9165         pipeline->n_ports_in = p->n_ports_in;
9166         pipeline->n_ports_out = p->n_ports_out;
9167         pipeline->n_actions = n_actions;
9168         pipeline->n_tables = n_tables;
9169         pipeline->n_selectors = p->n_selectors;
9170         pipeline->n_learners = p->n_learners;
9171         pipeline->n_regarrays = p->n_regarrays;
9172         pipeline->n_metarrays = p->n_metarrays;
9173
9174         return 0;
9175 }
9176
9177 int
9178 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9179 {
9180         if (!p || !numa_node)
9181                 return -EINVAL;
9182
9183         *numa_node = p->numa_node;
9184         return 0;
9185 }
9186
9187 int
9188 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9189                             uint32_t action_id,
9190                             struct rte_swx_ctl_action_info *action)
9191 {
9192         struct action *a = NULL;
9193
9194         if (!p || (action_id >= p->n_actions) || !action)
9195                 return -EINVAL;
9196
9197         a = action_find_by_id(p, action_id);
9198         if (!a)
9199                 return -EINVAL;
9200
9201         strcpy(action->name, a->name);
9202         action->n_args = a->st ? a->st->n_fields : 0;
9203         return 0;
9204 }
9205
9206 int
9207 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9208                                 uint32_t action_id,
9209                                 uint32_t action_arg_id,
9210                                 struct rte_swx_ctl_action_arg_info *action_arg)
9211 {
9212         struct action *a = NULL;
9213         struct field *arg = NULL;
9214
9215         if (!p || (action_id >= p->n_actions) || !action_arg)
9216                 return -EINVAL;
9217
9218         a = action_find_by_id(p, action_id);
9219         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9220                 return -EINVAL;
9221
9222         arg = &a->st->fields[action_arg_id];
9223         strcpy(action_arg->name, arg->name);
9224         action_arg->n_bits = arg->n_bits;
9225         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9226
9227         return 0;
9228 }
9229
9230 int
9231 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9232                            uint32_t table_id,
9233                            struct rte_swx_ctl_table_info *table)
9234 {
9235         struct table *t = NULL;
9236
9237         if (!p || !table)
9238                 return -EINVAL;
9239
9240         t = table_find_by_id(p, table_id);
9241         if (!t)
9242                 return -EINVAL;
9243
9244         strcpy(table->name, t->name);
9245         strcpy(table->args, t->args);
9246         table->n_match_fields = t->n_fields;
9247         table->n_actions = t->n_actions;
9248         table->default_action_is_const = t->default_action_is_const;
9249         table->size = t->size;
9250         return 0;
9251 }
9252
9253 int
9254 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9255         uint32_t table_id,
9256         uint32_t match_field_id,
9257         struct rte_swx_ctl_table_match_field_info *match_field)
9258 {
9259         struct table *t;
9260         struct match_field *f;
9261
9262         if (!p || (table_id >= p->n_tables) || !match_field)
9263                 return -EINVAL;
9264
9265         t = table_find_by_id(p, table_id);
9266         if (!t || (match_field_id >= t->n_fields))
9267                 return -EINVAL;
9268
9269         f = &t->fields[match_field_id];
9270         match_field->match_type = f->match_type;
9271         match_field->is_header = t->header ? 1 : 0;
9272         match_field->n_bits = f->field->n_bits;
9273         match_field->offset = f->field->offset;
9274
9275         return 0;
9276 }
9277
9278 int
9279 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9280         uint32_t table_id,
9281         uint32_t table_action_id,
9282         struct rte_swx_ctl_table_action_info *table_action)
9283 {
9284         struct table *t;
9285
9286         if (!p || (table_id >= p->n_tables) || !table_action)
9287                 return -EINVAL;
9288
9289         t = table_find_by_id(p, table_id);
9290         if (!t || (table_action_id >= t->n_actions))
9291                 return -EINVAL;
9292
9293         table_action->action_id = t->actions[table_action_id]->id;
9294
9295         return 0;
9296 }
9297
9298 int
9299 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9300                           uint32_t table_id,
9301                           struct rte_swx_table_ops *table_ops,
9302                           int *is_stub)
9303 {
9304         struct table *t;
9305
9306         if (!p || (table_id >= p->n_tables))
9307                 return -EINVAL;
9308
9309         t = table_find_by_id(p, table_id);
9310         if (!t)
9311                 return -EINVAL;
9312
9313         if (t->type) {
9314                 if (table_ops)
9315                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9316                 *is_stub = 0;
9317         } else {
9318                 *is_stub = 1;
9319         }
9320
9321         return 0;
9322 }
9323
9324 int
9325 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9326                               uint32_t selector_id,
9327                               struct rte_swx_ctl_selector_info *selector)
9328 {
9329         struct selector *s = NULL;
9330
9331         if (!p || !selector)
9332                 return -EINVAL;
9333
9334         s = selector_find_by_id(p, selector_id);
9335         if (!s)
9336                 return -EINVAL;
9337
9338         strcpy(selector->name, s->name);
9339
9340         selector->n_selector_fields = s->n_selector_fields;
9341         selector->n_groups_max = s->n_groups_max;
9342         selector->n_members_per_group_max = s->n_members_per_group_max;
9343
9344         return 0;
9345 }
9346
9347 int
9348 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9349          uint32_t selector_id,
9350          struct rte_swx_ctl_table_match_field_info *field)
9351 {
9352         struct selector *s;
9353
9354         if (!p || (selector_id >= p->n_selectors) || !field)
9355                 return -EINVAL;
9356
9357         s = selector_find_by_id(p, selector_id);
9358         if (!s)
9359                 return -EINVAL;
9360
9361         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9362         field->is_header = 0;
9363         field->n_bits = s->group_id_field->n_bits;
9364         field->offset = s->group_id_field->offset;
9365
9366         return 0;
9367 }
9368
9369 int
9370 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9371          uint32_t selector_id,
9372          uint32_t selector_field_id,
9373          struct rte_swx_ctl_table_match_field_info *field)
9374 {
9375         struct selector *s;
9376         struct field *f;
9377
9378         if (!p || (selector_id >= p->n_selectors) || !field)
9379                 return -EINVAL;
9380
9381         s = selector_find_by_id(p, selector_id);
9382         if (!s || (selector_field_id >= s->n_selector_fields))
9383                 return -EINVAL;
9384
9385         f = s->selector_fields[selector_field_id];
9386         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9387         field->is_header = s->selector_header ? 1 : 0;
9388         field->n_bits = f->n_bits;
9389         field->offset = f->offset;
9390
9391         return 0;
9392 }
9393
9394 int
9395 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9396          uint32_t selector_id,
9397          struct rte_swx_ctl_table_match_field_info *field)
9398 {
9399         struct selector *s;
9400
9401         if (!p || (selector_id >= p->n_selectors) || !field)
9402                 return -EINVAL;
9403
9404         s = selector_find_by_id(p, selector_id);
9405         if (!s)
9406                 return -EINVAL;
9407
9408         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9409         field->is_header = 0;
9410         field->n_bits = s->member_id_field->n_bits;
9411         field->offset = s->member_id_field->offset;
9412
9413         return 0;
9414 }
9415
9416 int
9417 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9418                              uint32_t learner_id,
9419                              struct rte_swx_ctl_learner_info *learner)
9420 {
9421         struct learner *l = NULL;
9422
9423         if (!p || !learner)
9424                 return -EINVAL;
9425
9426         l = learner_find_by_id(p, learner_id);
9427         if (!l)
9428                 return -EINVAL;
9429
9430         strcpy(learner->name, l->name);
9431
9432         learner->n_match_fields = l->n_fields;
9433         learner->n_actions = l->n_actions;
9434         learner->default_action_is_const = l->default_action_is_const;
9435         learner->size = l->size;
9436
9437         return 0;
9438 }
9439
9440 int
9441 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9442                                          uint32_t learner_id,
9443                                          uint32_t match_field_id,
9444                                          struct rte_swx_ctl_table_match_field_info *match_field)
9445 {
9446         struct learner *l;
9447         struct field *f;
9448
9449         if (!p || (learner_id >= p->n_learners) || !match_field)
9450                 return -EINVAL;
9451
9452         l = learner_find_by_id(p, learner_id);
9453         if (!l || (match_field_id >= l->n_fields))
9454                 return -EINVAL;
9455
9456         f = l->fields[match_field_id];
9457         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9458         match_field->is_header = l->header ? 1 : 0;
9459         match_field->n_bits = f->n_bits;
9460         match_field->offset = f->offset;
9461
9462         return 0;
9463 }
9464
9465 int
9466 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9467                                     uint32_t learner_id,
9468                                     uint32_t learner_action_id,
9469                                     struct rte_swx_ctl_table_action_info *learner_action)
9470 {
9471         struct learner *l;
9472
9473         if (!p || (learner_id >= p->n_learners) || !learner_action)
9474                 return -EINVAL;
9475
9476         l = learner_find_by_id(p, learner_id);
9477         if (!l || (learner_action_id >= l->n_actions))
9478                 return -EINVAL;
9479
9480         learner_action->action_id = l->actions[learner_action_id]->id;
9481
9482         return 0;
9483 }
9484
9485 int
9486 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9487                                  struct rte_swx_table_state **table_state)
9488 {
9489         if (!p || !table_state || !p->build_done)
9490                 return -EINVAL;
9491
9492         *table_state = p->table_state;
9493         return 0;
9494 }
9495
9496 int
9497 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9498                                  struct rte_swx_table_state *table_state)
9499 {
9500         if (!p || !table_state || !p->build_done)
9501                 return -EINVAL;
9502
9503         p->table_state = table_state;
9504         return 0;
9505 }
9506
9507 int
9508 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9509                                         uint32_t port_id,
9510                                         struct rte_swx_port_in_stats *stats)
9511 {
9512         struct port_in *port;
9513
9514         if (!p || !stats)
9515                 return -EINVAL;
9516
9517         port = port_in_find(p, port_id);
9518         if (!port)
9519                 return -EINVAL;
9520
9521         port->type->ops.stats_read(port->obj, stats);
9522         return 0;
9523 }
9524
9525 int
9526 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9527                                          uint32_t port_id,
9528                                          struct rte_swx_port_out_stats *stats)
9529 {
9530         struct port_out *port;
9531
9532         if (!p || !stats)
9533                 return -EINVAL;
9534
9535         port = port_out_find(p, port_id);
9536         if (!port)
9537                 return -EINVAL;
9538
9539         port->type->ops.stats_read(port->obj, stats);
9540         return 0;
9541 }
9542
9543 int
9544 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9545                                       const char *table_name,
9546                                       struct rte_swx_table_stats *stats)
9547 {
9548         struct table *table;
9549         struct table_statistics *table_stats;
9550
9551         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9552                 return -EINVAL;
9553
9554         table = table_find(p, table_name);
9555         if (!table)
9556                 return -EINVAL;
9557
9558         table_stats = &p->table_stats[table->id];
9559
9560         memcpy(stats->n_pkts_action,
9561                table_stats->n_pkts_action,
9562                p->n_actions * sizeof(uint64_t));
9563
9564         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9565         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9566
9567         return 0;
9568 }
9569
9570 int
9571 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9572         const char *selector_name,
9573         struct rte_swx_pipeline_selector_stats *stats)
9574 {
9575         struct selector *s;
9576
9577         if (!p || !selector_name || !selector_name[0] || !stats)
9578                 return -EINVAL;
9579
9580         s = selector_find(p, selector_name);
9581         if (!s)
9582                 return -EINVAL;
9583
9584         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9585
9586         return 0;
9587 }
9588
9589 int
9590 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9591                                         const char *learner_name,
9592                                         struct rte_swx_learner_stats *stats)
9593 {
9594         struct learner *l;
9595         struct learner_statistics *learner_stats;
9596
9597         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9598                 return -EINVAL;
9599
9600         l = learner_find(p, learner_name);
9601         if (!l)
9602                 return -EINVAL;
9603
9604         learner_stats = &p->learner_stats[l->id];
9605
9606         memcpy(stats->n_pkts_action,
9607                learner_stats->n_pkts_action,
9608                p->n_actions * sizeof(uint64_t));
9609
9610         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9611         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9612
9613         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9614         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9615
9616         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9617
9618         return 0;
9619 }
9620
9621 int
9622 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9623                               uint32_t regarray_id,
9624                               struct rte_swx_ctl_regarray_info *regarray)
9625 {
9626         struct regarray *r;
9627
9628         if (!p || !regarray)
9629                 return -EINVAL;
9630
9631         r = regarray_find_by_id(p, regarray_id);
9632         if (!r)
9633                 return -EINVAL;
9634
9635         strcpy(regarray->name, r->name);
9636         regarray->size = r->size;
9637         return 0;
9638 }
9639
9640 int
9641 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9642                                    const char *regarray_name,
9643                                    uint32_t regarray_index,
9644                                    uint64_t *value)
9645 {
9646         struct regarray *regarray;
9647         struct regarray_runtime *r;
9648
9649         if (!p || !regarray_name || !value)
9650                 return -EINVAL;
9651
9652         regarray = regarray_find(p, regarray_name);
9653         if (!regarray || (regarray_index >= regarray->size))
9654                 return -EINVAL;
9655
9656         r = &p->regarray_runtime[regarray->id];
9657         *value = r->regarray[regarray_index];
9658         return 0;
9659 }
9660
9661 int
9662 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9663                                    const char *regarray_name,
9664                                    uint32_t regarray_index,
9665                                    uint64_t value)
9666 {
9667         struct regarray *regarray;
9668         struct regarray_runtime *r;
9669
9670         if (!p || !regarray_name)
9671                 return -EINVAL;
9672
9673         regarray = regarray_find(p, regarray_name);
9674         if (!regarray || (regarray_index >= regarray->size))
9675                 return -EINVAL;
9676
9677         r = &p->regarray_runtime[regarray->id];
9678         r->regarray[regarray_index] = value;
9679         return 0;
9680 }
9681
9682 int
9683 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9684                               uint32_t metarray_id,
9685                               struct rte_swx_ctl_metarray_info *metarray)
9686 {
9687         struct metarray *m;
9688
9689         if (!p || !metarray)
9690                 return -EINVAL;
9691
9692         m = metarray_find_by_id(p, metarray_id);
9693         if (!m)
9694                 return -EINVAL;
9695
9696         strcpy(metarray->name, m->name);
9697         metarray->size = m->size;
9698         return 0;
9699 }
9700
9701 int
9702 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9703                               const char *name,
9704                               struct rte_meter_trtcm_params *params)
9705 {
9706         struct meter_profile *mp;
9707         int status;
9708
9709         CHECK(p, EINVAL);
9710         CHECK_NAME(name, EINVAL);
9711         CHECK(params, EINVAL);
9712         CHECK(!meter_profile_find(p, name), EEXIST);
9713
9714         /* Node allocation. */
9715         mp = calloc(1, sizeof(struct meter_profile));
9716         CHECK(mp, ENOMEM);
9717
9718         /* Node initialization. */
9719         strcpy(mp->name, name);
9720         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9721         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9722         if (status) {
9723                 free(mp);
9724                 CHECK(0, EINVAL);
9725         }
9726
9727         /* Node add to tailq. */
9728         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9729
9730         return 0;
9731 }
9732
9733 int
9734 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9735                                  const char *name)
9736 {
9737         struct meter_profile *mp;
9738
9739         CHECK(p, EINVAL);
9740         CHECK_NAME(name, EINVAL);
9741
9742         mp = meter_profile_find(p, name);
9743         CHECK(mp, EINVAL);
9744         CHECK(!mp->n_users, EBUSY);
9745
9746         /* Remove node from tailq. */
9747         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9748         free(mp);
9749
9750         return 0;
9751 }
9752
9753 int
9754 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9755                         const char *metarray_name,
9756                         uint32_t metarray_index)
9757 {
9758         struct meter_profile *mp_old;
9759         struct metarray *metarray;
9760         struct metarray_runtime *metarray_runtime;
9761         struct meter *m;
9762
9763         CHECK(p, EINVAL);
9764         CHECK_NAME(metarray_name, EINVAL);
9765
9766         metarray = metarray_find(p, metarray_name);
9767         CHECK(metarray, EINVAL);
9768         CHECK(metarray_index < metarray->size, EINVAL);
9769
9770         metarray_runtime = &p->metarray_runtime[metarray->id];
9771         m = &metarray_runtime->metarray[metarray_index];
9772         mp_old = m->profile;
9773
9774         meter_init(m);
9775
9776         mp_old->n_users--;
9777
9778         return 0;
9779 }
9780
9781 int
9782 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9783                       const char *metarray_name,
9784                       uint32_t metarray_index,
9785                       const char *profile_name)
9786 {
9787         struct meter_profile *mp, *mp_old;
9788         struct metarray *metarray;
9789         struct metarray_runtime *metarray_runtime;
9790         struct meter *m;
9791
9792         CHECK(p, EINVAL);
9793         CHECK_NAME(metarray_name, EINVAL);
9794
9795         metarray = metarray_find(p, metarray_name);
9796         CHECK(metarray, EINVAL);
9797         CHECK(metarray_index < metarray->size, EINVAL);
9798
9799         mp = meter_profile_find(p, profile_name);
9800         CHECK(mp, EINVAL);
9801
9802         metarray_runtime = &p->metarray_runtime[metarray->id];
9803         m = &metarray_runtime->metarray[metarray_index];
9804         mp_old = m->profile;
9805
9806         memset(m, 0, sizeof(struct meter));
9807         rte_meter_trtcm_config(&m->m, &mp->profile);
9808         m->profile = mp;
9809         m->color_mask = RTE_COLORS;
9810
9811         mp->n_users++;
9812         mp_old->n_users--;
9813
9814         return 0;
9815 }
9816
9817 int
9818 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9819                              const char *metarray_name,
9820                              uint32_t metarray_index,
9821                              struct rte_swx_ctl_meter_stats *stats)
9822 {
9823         struct metarray *metarray;
9824         struct metarray_runtime *metarray_runtime;
9825         struct meter *m;
9826
9827         CHECK(p, EINVAL);
9828         CHECK_NAME(metarray_name, EINVAL);
9829
9830         metarray = metarray_find(p, metarray_name);
9831         CHECK(metarray, EINVAL);
9832         CHECK(metarray_index < metarray->size, EINVAL);
9833
9834         CHECK(stats, EINVAL);
9835
9836         metarray_runtime = &p->metarray_runtime[metarray->id];
9837         m = &metarray_runtime->metarray[metarray_index];
9838
9839         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9840         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9841
9842         return 0;
9843 }
9844
9845 /*
9846  * Pipeline compilation.
9847  */
9848 static const char *
9849 instr_type_to_name(struct instruction *instr)
9850 {
9851         switch (instr->type) {
9852         case INSTR_RX: return "INSTR_RX";
9853
9854         case INSTR_TX: return "INSTR_TX";
9855         case INSTR_TX_I: return "INSTR_TX_I";
9856
9857         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
9858         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
9859         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
9860         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
9861         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
9862         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
9863         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
9864         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
9865
9866         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
9867
9868         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
9869
9870         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
9871         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
9872         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
9873         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
9874         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
9875         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
9876         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
9877         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
9878         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
9879
9880         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
9881         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
9882
9883         case INSTR_MOV: return "INSTR_MOV";
9884         case INSTR_MOV_MH: return "INSTR_MOV_MH";
9885         case INSTR_MOV_HM: return "INSTR_MOV_HM";
9886         case INSTR_MOV_HH: return "INSTR_MOV_HH";
9887         case INSTR_MOV_I: return "INSTR_MOV_I";
9888
9889         case INSTR_DMA_HT: return "INSTR_DMA_HT";
9890         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
9891         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
9892         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
9893         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
9894         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
9895         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
9896         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
9897
9898         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
9899         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
9900         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
9901         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
9902         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
9903         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
9904
9905         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
9906         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
9907         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
9908         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
9909         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
9910         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
9911
9912         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
9913         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
9914         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
9915         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
9916
9917         case INSTR_ALU_AND: return "INSTR_ALU_AND";
9918         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
9919         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
9920         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
9921         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
9922
9923         case INSTR_ALU_OR: return "INSTR_ALU_OR";
9924         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
9925         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
9926         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
9927         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
9928
9929         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
9930         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
9931         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
9932         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
9933         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
9934
9935         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
9936         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
9937         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
9938         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
9939         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
9940         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
9941
9942         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
9943         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
9944         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
9945         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
9946         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
9947         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
9948
9949         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
9950         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
9951         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
9952
9953         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
9954         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
9955         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
9956         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
9957         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
9958         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
9959
9960         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
9961         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
9962         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
9963         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
9964         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
9965         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
9966         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
9967         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
9968         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
9969
9970         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
9971         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
9972         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
9973         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
9974         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
9975         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
9976         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
9977         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
9978         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
9979
9980         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
9981         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
9982         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
9983
9984         case INSTR_METER_HHM: return "INSTR_METER_HHM";
9985         case INSTR_METER_HHI: return "INSTR_METER_HHI";
9986         case INSTR_METER_HMM: return "INSTR_METER_HMM";
9987         case INSTR_METER_HMI: return "INSTR_METER_HMI";
9988         case INSTR_METER_MHM: return "INSTR_METER_MHM";
9989         case INSTR_METER_MHI: return "INSTR_METER_MHI";
9990         case INSTR_METER_MMM: return "INSTR_METER_MMM";
9991         case INSTR_METER_MMI: return "INSTR_METER_MMI";
9992         case INSTR_METER_IHM: return "INSTR_METER_IHM";
9993         case INSTR_METER_IHI: return "INSTR_METER_IHI";
9994         case INSTR_METER_IMM: return "INSTR_METER_IMM";
9995         case INSTR_METER_IMI: return "INSTR_METER_IMI";
9996
9997         case INSTR_TABLE: return "INSTR_TABLE";
9998         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
9999         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10000         case INSTR_LEARNER: return "INSTR_LEARNER";
10001         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10002
10003         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10004         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10005
10006         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10007         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10008
10009         case INSTR_JMP: return "INSTR_JMP";
10010         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10011         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10012         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10013         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10014         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10015         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10016         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10017         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10018         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10019         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10020         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10021         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10022         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10023         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10024         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10025         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10026         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10027         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10028         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10029         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10030         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10031         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10032         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10033         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10034         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10035         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10036         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10037         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10038
10039         case INSTR_RETURN: return "INSTR_RETURN";
10040
10041         default: return "INSTR_UNKNOWN";
10042         }
10043 }
10044
10045 typedef void
10046 (*instruction_export_t)(struct instruction *, FILE *);
10047
10048 static void
10049 instr_io_export(struct instruction *instr, FILE *f)
10050 {
10051         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10052
10053         /* n_io, n_io_imm, n_hdrs. */
10054         if (instr->type == INSTR_RX ||
10055             instr->type == INSTR_TX ||
10056             instr->type == INSTR_HDR_EXTRACT_M ||
10057             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10058                 n_io = 1;
10059
10060         if (instr->type == INSTR_TX_I)
10061                 n_io_imm = 1;
10062
10063         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10064                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10065
10066         if (instr->type == INSTR_HDR_EXTRACT_M ||
10067             instr->type == INSTR_HDR_LOOKAHEAD ||
10068             instr->type == INSTR_HDR_EMIT)
10069                 n_hdrs = 1;
10070
10071         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10072                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10073
10074         /* instr. */
10075         fprintf(f,
10076                 "\t{\n"
10077                 "\t\t.type = %s,\n",
10078                 instr_type_to_name(instr));
10079
10080         /* instr.io. */
10081         fprintf(f,
10082                 "\t\t.io = {\n");
10083
10084         /* instr.io.io. */
10085         if (n_io)
10086                 fprintf(f,
10087                         "\t\t\t.io = {\n"
10088                         "\t\t\t\t.offset = %u,\n"
10089                         "\t\t\t\t.n_bits = %u,\n"
10090                         "\t\t\t},\n",
10091                         instr->io.io.offset,
10092                         instr->io.io.n_bits);
10093
10094         if (n_io_imm)
10095                 fprintf(f,
10096                         "\t\t\t.io = {\n"
10097                         "\t\t\t\t.val = %u,\n"
10098                         "\t\t\t},\n",
10099                         instr->io.io.val);
10100
10101         /* instr.io.hdr. */
10102         if (n_hdrs) {
10103                 fprintf(f,
10104                         "\t\t.hdr = {\n");
10105
10106                 /* instr.io.hdr.header_id. */
10107                 fprintf(f,
10108                         "\t\t\t.header_id = {");
10109
10110                 for (i = 0; i < n_hdrs; i++)
10111                         fprintf(f,
10112                                 "%u, ",
10113                                 instr->io.hdr.header_id[i]);
10114
10115                 fprintf(f,
10116                         "},\n");
10117
10118                 /* instr.io.hdr.struct_id. */
10119                 fprintf(f,
10120                         "\t\t\t.struct_id = {");
10121
10122                 for (i = 0; i < n_hdrs; i++)
10123                         fprintf(f,
10124                                 "%u, ",
10125                                 instr->io.hdr.struct_id[i]);
10126
10127                 fprintf(f,
10128                         "},\n");
10129
10130                 /* instr.io.hdr.n_bytes. */
10131                 fprintf(f,
10132                         "\t\t\t.n_bytes = {");
10133
10134                 for (i = 0; i < n_hdrs; i++)
10135                         fprintf(f,
10136                                 "%u, ",
10137                                 instr->io.hdr.n_bytes[i]);
10138
10139                 fprintf(f,
10140                         "},\n");
10141
10142                 /* instr.io.hdr - closing curly brace. */
10143                 fprintf(f,
10144                         "\t\t\t}\n,");
10145         }
10146
10147         /* instr.io - closing curly brace. */
10148         fprintf(f,
10149                 "\t\t},\n");
10150
10151         /* instr - closing curly brace. */
10152         fprintf(f,
10153                 "\t},\n");
10154 }
10155
10156 static void
10157 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10158 {
10159         fprintf(f,
10160                 "\t{\n"
10161                 "\t\t.type = %s,\n"
10162                 "\t\t.valid = {\n"
10163                 "\t\t\t.header_id = %u,\n"
10164                 "\t\t},\n"
10165                 "\t},\n",
10166                 instr_type_to_name(instr),
10167                 instr->valid.header_id);
10168 }
10169
10170 static void
10171 instr_mov_export(struct instruction *instr, FILE *f)
10172 {
10173         if (instr->type != INSTR_MOV_I)
10174                 fprintf(f,
10175                         "\t{\n"
10176                         "\t\t.type = %s,\n"
10177                         "\t\t.mov = {\n"
10178                         "\t\t\t.dst = {\n"
10179                         "\t\t\t\t.struct_id = %u,\n"
10180                         "\t\t\t\t.n_bits = %u,\n"
10181                         "\t\t\t\t.offset = %u,\n"
10182                         "\t\t\t},\n"
10183                         "\t\t\t.src = {\n"
10184                         "\t\t\t\t.struct_id = %u,\n"
10185                         "\t\t\t\t.n_bits = %u,\n"
10186                         "\t\t\t\t.offset = %u,\n"
10187                         "\t\t\t},\n"
10188                         "\t\t},\n"
10189                         "\t},\n",
10190                         instr_type_to_name(instr),
10191                         instr->mov.dst.struct_id,
10192                         instr->mov.dst.n_bits,
10193                         instr->mov.dst.offset,
10194                         instr->mov.src.struct_id,
10195                         instr->mov.src.n_bits,
10196                         instr->mov.src.offset);
10197         else
10198                 fprintf(f,
10199                         "\t{\n"
10200                         "\t\t.type = %s,\n"
10201                         "\t\t.mov = {\n"
10202                         "\t\t\t.dst = {\n"
10203                         "\t\t\t\t.struct_id = %u,\n"
10204                         "\t\t\t\t.n_bits = %u,\n"
10205                         "\t\t\t\t.offset = %u,\n"
10206                         "\t\t\t}\n,"
10207                         "\t\t\t.src_val = %" PRIu64 ",\n"
10208                         "\t\t},\n"
10209                         "\t},\n",
10210                         instr_type_to_name(instr),
10211                         instr->mov.dst.struct_id,
10212                         instr->mov.dst.n_bits,
10213                         instr->mov.dst.offset,
10214                         instr->mov.src_val);
10215 }
10216
10217 static void
10218 instr_dma_ht_export(struct instruction *instr, FILE *f)
10219 {
10220         uint32_t n_dma = 0, i;
10221
10222         /* n_dma. */
10223         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10224
10225         /* instr. */
10226         fprintf(f,
10227                 "\t{\n"
10228                 "\t\t.type = %s,\n",
10229                 instr_type_to_name(instr));
10230
10231         /* instr.dma. */
10232         fprintf(f,
10233                 "\t\t.dma = {\n");
10234
10235         /* instr.dma.dst. */
10236         fprintf(f,
10237                 "\t\t\t.dst = {\n");
10238
10239         /* instr.dma.dst.header_id. */
10240         fprintf(f,
10241                 "\t\t\t\t.header_id = {");
10242
10243         for (i = 0; i < n_dma; i++)
10244                 fprintf(f,
10245                         "%u, ",
10246                         instr->dma.dst.header_id[i]);
10247
10248         fprintf(f,
10249                 "},\n");
10250
10251         /* instr.dma.dst.struct_id. */
10252         fprintf(f,
10253                 "\t\t\t\t.struct_id = {");
10254
10255         for (i = 0; i < n_dma; i++)
10256                 fprintf(f,
10257                         "%u, ",
10258                         instr->dma.dst.struct_id[i]);
10259
10260         fprintf(f,
10261                 "},\n");
10262
10263         /* instr.dma.dst - closing curly brace. */
10264         fprintf(f,
10265                 "\t\t\t},\n");
10266
10267         /* instr.dma.src. */
10268         fprintf(f,
10269                 "\t\t\t.src = {\n");
10270
10271         /* instr.dma.src.offset. */
10272         fprintf(f,
10273                 "\t\t\t\t.offset = {");
10274
10275         for (i = 0; i < n_dma; i++)
10276                 fprintf(f,
10277                         "%u, ",
10278                         instr->dma.src.offset[i]);
10279
10280         fprintf(f,
10281                 "},\n");
10282
10283         /* instr.dma.src - closing curly brace. */
10284         fprintf(f,
10285                 "\t\t\t},\n");
10286
10287         /* instr.dma.n_bytes. */
10288         fprintf(f,
10289                 "\t\t\t.n_bytes = {");
10290
10291         for (i = 0; i < n_dma; i++)
10292                 fprintf(f,
10293                         "%u, ",
10294                         instr->dma.n_bytes[i]);
10295
10296         fprintf(f,
10297                 "},\n");
10298
10299         /* instr.dma - closing curly brace. */
10300         fprintf(f,
10301                 "\t\t},\n");
10302
10303         /* instr - closing curly brace. */
10304         fprintf(f,
10305                 "\t},\n");
10306 }
10307
10308 static void
10309 instr_alu_export(struct instruction *instr, FILE *f)
10310 {
10311         int imm = 0;
10312
10313         if (instr->type == INSTR_ALU_ADD_MI ||
10314             instr->type == INSTR_ALU_ADD_HI ||
10315             instr->type == INSTR_ALU_SUB_MI ||
10316             instr->type == INSTR_ALU_SUB_HI ||
10317             instr->type == INSTR_ALU_SHL_MI ||
10318             instr->type == INSTR_ALU_SHL_HI ||
10319             instr->type == INSTR_ALU_SHR_MI ||
10320             instr->type == INSTR_ALU_SHR_HI ||
10321             instr->type == INSTR_ALU_AND_I ||
10322             instr->type == INSTR_ALU_OR_I ||
10323             instr->type == INSTR_ALU_XOR_I)
10324                 imm = 1;
10325
10326         if (!imm)
10327                 fprintf(f,
10328                         "\t{\n"
10329                         "\t\t.type = %s,\n"
10330                         "\t\t.alu = {\n"
10331                         "\t\t\t.dst = {\n"
10332                         "\t\t\t\t.struct_id = %u,\n"
10333                         "\t\t\t\t.n_bits = %u,\n"
10334                         "\t\t\t\t.offset = %u,\n"
10335                         "\t\t\t},\n"
10336                         "\t\t\t.src = {\n"
10337                         "\t\t\t\t.struct_id = %u,\n"
10338                         "\t\t\t\t.n_bits = %u,\n"
10339                         "\t\t\t\t.offset = %u,\n"
10340                         "\t\t\t},\n"
10341                         "\t\t},\n"
10342                         "\t},\n",
10343                         instr_type_to_name(instr),
10344                         instr->alu.dst.struct_id,
10345                         instr->alu.dst.n_bits,
10346                         instr->alu.dst.offset,
10347                         instr->alu.src.struct_id,
10348                         instr->alu.src.n_bits,
10349                         instr->alu.src.offset);
10350         else
10351                 fprintf(f,
10352                         "\t{\n"
10353                         "\t\t.type = %s,\n"
10354                         "\t\t.alu = {\n"
10355                         "\t\t\t.dst = {\n"
10356                         "\t\t\t\t.struct_id = %u,\n"
10357                         "\t\t\t\t.n_bits = %u,\n"
10358                         "\t\t\t\t.offset = %u,\n"
10359                         "\t\t\t}\n,"
10360                         "\t\t\t.src_val = %" PRIu64 ",\n"
10361                         "\t\t},\n"
10362                         "\t},\n",
10363                         instr_type_to_name(instr),
10364                         instr->alu.dst.struct_id,
10365                         instr->alu.dst.n_bits,
10366                         instr->alu.dst.offset,
10367                         instr->alu.src_val);
10368 }
10369
10370 static void
10371 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10372 {
10373         int prefetch  = 0, idx_imm = 0, src_imm = 0;
10374
10375         if (instr->type == INSTR_REGPREFETCH_RH ||
10376             instr->type == INSTR_REGPREFETCH_RM ||
10377             instr->type == INSTR_REGPREFETCH_RI)
10378                 prefetch = 1;
10379
10380         /* index is the 3rd operand for the regrd instruction and the 2nd
10381          * operand for the regwr and regadd instructions.
10382          */
10383         if (instr->type == INSTR_REGPREFETCH_RI ||
10384             instr->type == INSTR_REGRD_HRI ||
10385             instr->type == INSTR_REGRD_MRI ||
10386             instr->type == INSTR_REGWR_RIH ||
10387             instr->type == INSTR_REGWR_RIM ||
10388             instr->type == INSTR_REGWR_RII ||
10389             instr->type == INSTR_REGADD_RIH ||
10390             instr->type == INSTR_REGADD_RIM ||
10391             instr->type == INSTR_REGADD_RII)
10392                 idx_imm = 1;
10393
10394         /* src is the 3rd operand for the regwr and regadd instructions. */
10395         if (instr->type == INSTR_REGWR_RHI ||
10396             instr->type == INSTR_REGWR_RMI ||
10397             instr->type == INSTR_REGWR_RII ||
10398             instr->type == INSTR_REGADD_RHI ||
10399             instr->type == INSTR_REGADD_RMI ||
10400             instr->type == INSTR_REGADD_RII)
10401                 src_imm = 1;
10402
10403         /* instr.regarray.regarray_id. */
10404         fprintf(f,
10405                 "\t{\n"
10406                 "\t\t.type = %s,\n"
10407                 "\t\t.regarray = {\n"
10408                 "\t\t\t.regarray_id = %u,\n",
10409                 instr_type_to_name(instr),
10410                 instr->regarray.regarray_id);
10411
10412         /* instr.regarray.idx / instr.regarray.idx_val. */
10413         if (!idx_imm)
10414                 fprintf(f,
10415                         "\t\t\t\t.idx = {\n"
10416                         "\t\t\t\t\t.struct_id = %u,\n"
10417                         "\t\t\t\t\t.n_bits = %u,\n"
10418                         "\t\t\t\t\t.offset = %u,\n"
10419                         "\t\t\t\t},\n",
10420                         instr->regarray.idx.struct_id,
10421                         instr->regarray.idx.n_bits,
10422                         instr->regarray.idx.offset);
10423         else
10424                 fprintf(f,
10425                         "\t\t\t\t.idx_val = %u,\n",
10426                         instr->regarray.idx_val);
10427
10428         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10429         if (!prefetch) {
10430                 if (!src_imm)
10431                         fprintf(f,
10432                                 "\t\t\t\t.dstsrc = {\n"
10433                                 "\t\t\t\t\t.struct_id = %u,\n"
10434                                 "\t\t\t\t\t.n_bits = %u,\n"
10435                                 "\t\t\t\t\t.offset = %u,\n"
10436                                 "\t\t\t\t},\n",
10437                                 instr->regarray.dstsrc.struct_id,
10438                                 instr->regarray.dstsrc.n_bits,
10439                                 instr->regarray.dstsrc.offset);
10440                 else
10441                         fprintf(f,
10442                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10443                                 instr->regarray.dstsrc_val);
10444         }
10445
10446         /* instr.regarray and instr - closing curly braces. */
10447         fprintf(f,
10448                 "\t\t},\n"
10449                 "\t},\n");
10450 }
10451
10452 static void
10453 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10454 {
10455         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
10456
10457         if (instr->type == INSTR_METPREFETCH_H ||
10458             instr->type == INSTR_METPREFETCH_M ||
10459             instr->type == INSTR_METPREFETCH_I)
10460                 prefetch = 1;
10461
10462         /* idx_imm. */
10463         if (instr->type == INSTR_METPREFETCH_I ||
10464             instr->type == INSTR_METER_IHM ||
10465             instr->type == INSTR_METER_IHI ||
10466             instr->type == INSTR_METER_IMM ||
10467             instr->type == INSTR_METER_IMI)
10468                 idx_imm = 1;
10469
10470         /* color_in_imm. */
10471         if (instr->type == INSTR_METER_HHI ||
10472             instr->type == INSTR_METER_HMI ||
10473             instr->type == INSTR_METER_MHI ||
10474             instr->type == INSTR_METER_MMI ||
10475             instr->type == INSTR_METER_IHI ||
10476             instr->type == INSTR_METER_IMI)
10477                 color_in_imm = 1;
10478
10479         /* instr.meter.metarray_id. */
10480         fprintf(f,
10481                 "\t{\n"
10482                 "\t\t.type = %s,\n"
10483                 "\t\t.meter = {\n"
10484                 "\t\t\t.metarray_id = %u,\n",
10485                 instr_type_to_name(instr),
10486                 instr->meter.metarray_id);
10487
10488         /* instr.meter.idx / instr.meter.idx_val. */
10489         if (!idx_imm)
10490                 fprintf(f,
10491                         "\t\t\t.idx = {\n"
10492                         "\t\t\t\t.struct_id = %u,\n"
10493                         "\t\t\t\t.n_bits = %u,\n"
10494                         "\t\t\t\t.offset = %u,\n"
10495                         "\t\t\t},\n",
10496                         instr->meter.idx.struct_id,
10497                         instr->meter.idx.n_bits,
10498                         instr->meter.idx.offset);
10499         else
10500                 fprintf(f,
10501                         "\t\t\t.idx_val = %u,\n",
10502                         instr->meter.idx_val);
10503
10504         if (!prefetch) {
10505                 /* instr.meter.length. */
10506                 fprintf(f,
10507                         "\t\t\t.length = {\n"
10508                         "\t\t\t\t.struct_id = %u,\n"
10509                         "\t\t\t\t.n_bits = %u,\n"
10510                         "\t\t\t\t.offset = %u,\n"
10511                         "\t\t\t},\n",
10512                         instr->meter.length.struct_id,
10513                         instr->meter.length.n_bits,
10514                         instr->meter.length.offset);
10515
10516                 /* instr.meter.color_in / instr.meter.color_in_val. */
10517                 if (!color_in_imm)
10518                         fprintf(f,
10519                                 "\t\t\t.color_in = {\n"
10520                                 "\t\t\t\t.struct_id = %u,\n"
10521                                 "\t\t\t\t.n_bits = %u,\n"
10522                                 "\t\t\t\t.offset = %u,\n"
10523                                 "\t\t\t},\n",
10524                                 instr->meter.color_in.struct_id,
10525                                 instr->meter.color_in.n_bits,
10526                                 instr->meter.color_in.offset);
10527                 else
10528                         fprintf(f,
10529                                 "\t\t\t.color_in_val = %u,\n",
10530                                 (uint32_t)instr->meter.color_in_val);
10531
10532                 /* instr.meter.color_out. */
10533                 fprintf(f,
10534                         "\t\t\t.color_out = {\n"
10535                         "\t\t\t\t.struct_id = %u,\n"
10536                         "\t\t\t\t.n_bits = %u,\n"
10537                         "\t\t\t\t.offset = %u,\n"
10538                         "\t\t\t},\n",
10539                         instr->meter.color_out.struct_id,
10540                         instr->meter.color_out.n_bits,
10541                         instr->meter.color_out.offset);
10542         }
10543
10544         /* instr.meter and instr - closing curly braces. */
10545         fprintf(f,
10546                 "\t\t},\n"
10547                 "\t},\n");
10548 }
10549
10550 static void
10551 instr_table_export(struct instruction *instr,
10552                 FILE *f)
10553 {
10554         fprintf(f,
10555                 "\t{\n"
10556                 "\t\t.type = %s,\n"
10557                 "\t\t.table = {\n"
10558                 "\t\t\t.table_id = %u,\n"
10559                 "\t\t},\n"
10560                 "\t},\n",
10561                 instr_type_to_name(instr),
10562                 instr->table.table_id);
10563 }
10564
10565 static void
10566 instr_learn_export(struct instruction *instr, FILE *f)
10567 {
10568         fprintf(f,
10569                 "\t{\n"
10570                 "\t\t.type = %s,\n"
10571                 "\t\t.learn = {\n"
10572                 "\t\t\t\t.action_id = %u,\n"
10573                 "\t\t},\n"
10574                 "\t},\n",
10575                 instr_type_to_name(instr),
10576                 instr->learn.action_id);
10577 }
10578
10579 static void
10580 instr_forget_export(struct instruction *instr, FILE *f)
10581 {
10582         fprintf(f,
10583                 "\t{\n"
10584                 "\t\t.type = %s,\n"
10585                 "\t},\n",
10586                 instr_type_to_name(instr));
10587 }
10588
10589 static void
10590 instr_extern_export(struct instruction *instr, FILE *f)
10591 {
10592         if (instr->type == INSTR_EXTERN_OBJ)
10593                 fprintf(f,
10594                         "\t{\n"
10595                         "\t\t.type = %s,\n"
10596                         "\t\t.ext_obj = {\n"
10597                         "\t\t\t.ext_obj_id = %u,\n"
10598                         "\t\t\t.func_id = %u,\n"
10599                         "\t\t},\n"
10600                         "\t},\n",
10601                         instr_type_to_name(instr),
10602                         instr->ext_obj.ext_obj_id,
10603                         instr->ext_obj.func_id);
10604         else
10605                 fprintf(f,
10606                         "\t{\n"
10607                         "\t\t.type = %s,\n"
10608                         "\t\t.ext_func = {\n"
10609                         "\t\t\t.ext_func_id = %u,\n"
10610                         "\t\t},\n"
10611                         "\t},\n",
10612                         instr_type_to_name(instr),
10613                         instr->ext_func.ext_func_id);
10614 }
10615
10616 static void
10617 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10618 {
10619         fprintf(f,
10620                 "\t{\n"
10621                 "\t\t.type = %s,\n"
10622                 "\t\t.jmp = {\n"
10623                 "\t\t\t.ip = NULL,\n",
10624                 instr_type_to_name(instr));
10625
10626         switch (instr->type) {
10627         case INSTR_JMP_VALID:
10628         case INSTR_JMP_INVALID:
10629                 fprintf(f,
10630                         "\t\t\t.header_id = %u,\n",
10631                         instr->jmp.header_id);
10632                 break;
10633
10634         case INSTR_JMP_ACTION_HIT:
10635         case INSTR_JMP_ACTION_MISS:
10636                 fprintf(f,
10637                         "\t\t\t.action_id = %u,\n",
10638                         instr->jmp.action_id);
10639                 break;
10640
10641         case INSTR_JMP_EQ:
10642         case INSTR_JMP_EQ_MH:
10643         case INSTR_JMP_EQ_HM:
10644         case INSTR_JMP_EQ_HH:
10645         case INSTR_JMP_NEQ:
10646         case INSTR_JMP_NEQ_MH:
10647         case INSTR_JMP_NEQ_HM:
10648         case INSTR_JMP_NEQ_HH:
10649         case INSTR_JMP_LT:
10650         case INSTR_JMP_LT_MH:
10651         case INSTR_JMP_LT_HM:
10652         case INSTR_JMP_LT_HH:
10653         case INSTR_JMP_GT:
10654         case INSTR_JMP_GT_MH:
10655         case INSTR_JMP_GT_HM:
10656         case INSTR_JMP_GT_HH:
10657                 fprintf(f,
10658                         "\t\t\t.a = {\n"
10659                         "\t\t\t\t.struct_id = %u,\n"
10660                         "\t\t\t\t.n_bits = %u,\n"
10661                         "\t\t\t\t.offset = %u,\n"
10662                         "\t\t\t},\n"
10663                         "\t\t\t.b = {\n"
10664                         "\t\t\t\t.struct_id = %u,\n"
10665                         "\t\t\t\t.n_bits = %u,\n"
10666                         "\t\t\t\t.offset = %u,\n"
10667                         "\t\t\t},\n",
10668                         instr->jmp.a.struct_id,
10669                         instr->jmp.a.n_bits,
10670                         instr->jmp.a.offset,
10671                         instr->jmp.b.struct_id,
10672                         instr->jmp.b.n_bits,
10673                         instr->jmp.b.offset);
10674                 break;
10675
10676         case INSTR_JMP_EQ_I:
10677         case INSTR_JMP_NEQ_I:
10678         case INSTR_JMP_LT_MI:
10679         case INSTR_JMP_LT_HI:
10680         case INSTR_JMP_GT_MI:
10681         case INSTR_JMP_GT_HI:
10682                 fprintf(f,
10683                         "\t\t\t.a = {\n"
10684                         "\t\t\t\t.struct_id = %u,\n"
10685                         "\t\t\t\t.n_bits = %u,\n"
10686                         "\t\t\t\t.offset = %u,\n"
10687                         "\t\t\t}\n,"
10688                         "\t\t\t.b_val = %" PRIu64 ",\n",
10689                         instr->jmp.a.struct_id,
10690                         instr->jmp.a.n_bits,
10691                         instr->jmp.a.offset,
10692                         instr->jmp.b_val);
10693                 break;
10694
10695         default:
10696                 break;
10697         }
10698
10699         fprintf(f,
10700                 "\t\t},\n"
10701                 "\t},\n");
10702 }
10703
10704 static void
10705 instr_return_export(struct instruction *instr,
10706                 FILE *f)
10707 {
10708         fprintf(f,
10709                 "\t{\n"
10710                 "\t\t.type = %s,\n",
10711                 instr_type_to_name(instr));
10712
10713         fprintf(f,
10714                 "\t},\n");
10715 }
10716
10717 static instruction_export_t export_table[] = {
10718         [INSTR_RX] = instr_io_export,
10719
10720         [INSTR_TX] = instr_io_export,
10721         [INSTR_TX_I] = instr_io_export,
10722
10723         [INSTR_HDR_EXTRACT] = instr_io_export,
10724         [INSTR_HDR_EXTRACT2] = instr_io_export,
10725         [INSTR_HDR_EXTRACT3] = instr_io_export,
10726         [INSTR_HDR_EXTRACT4] = instr_io_export,
10727         [INSTR_HDR_EXTRACT5] = instr_io_export,
10728         [INSTR_HDR_EXTRACT6] = instr_io_export,
10729         [INSTR_HDR_EXTRACT7] = instr_io_export,
10730         [INSTR_HDR_EXTRACT8] = instr_io_export,
10731
10732         [INSTR_HDR_EXTRACT_M] = instr_io_export,
10733
10734         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10735
10736         [INSTR_HDR_EMIT] = instr_io_export,
10737         [INSTR_HDR_EMIT_TX] = instr_io_export,
10738         [INSTR_HDR_EMIT2_TX] = instr_io_export,
10739         [INSTR_HDR_EMIT3_TX] = instr_io_export,
10740         [INSTR_HDR_EMIT4_TX] = instr_io_export,
10741         [INSTR_HDR_EMIT5_TX] = instr_io_export,
10742         [INSTR_HDR_EMIT6_TX] = instr_io_export,
10743         [INSTR_HDR_EMIT7_TX] = instr_io_export,
10744         [INSTR_HDR_EMIT8_TX] = instr_io_export,
10745
10746         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10747         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10748
10749         [INSTR_MOV] = instr_mov_export,
10750         [INSTR_MOV_MH] = instr_mov_export,
10751         [INSTR_MOV_HM] = instr_mov_export,
10752         [INSTR_MOV_HH] = instr_mov_export,
10753         [INSTR_MOV_I] = instr_mov_export,
10754
10755         [INSTR_DMA_HT]  = instr_dma_ht_export,
10756         [INSTR_DMA_HT2] = instr_dma_ht_export,
10757         [INSTR_DMA_HT3] = instr_dma_ht_export,
10758         [INSTR_DMA_HT4] = instr_dma_ht_export,
10759         [INSTR_DMA_HT5] = instr_dma_ht_export,
10760         [INSTR_DMA_HT6] = instr_dma_ht_export,
10761         [INSTR_DMA_HT7] = instr_dma_ht_export,
10762         [INSTR_DMA_HT8] = instr_dma_ht_export,
10763
10764         [INSTR_ALU_ADD] = instr_alu_export,
10765         [INSTR_ALU_ADD_MH] = instr_alu_export,
10766         [INSTR_ALU_ADD_HM] = instr_alu_export,
10767         [INSTR_ALU_ADD_HH] = instr_alu_export,
10768         [INSTR_ALU_ADD_MI] = instr_alu_export,
10769         [INSTR_ALU_ADD_HI] = instr_alu_export,
10770
10771         [INSTR_ALU_SUB] = instr_alu_export,
10772         [INSTR_ALU_SUB_MH] = instr_alu_export,
10773         [INSTR_ALU_SUB_HM] = instr_alu_export,
10774         [INSTR_ALU_SUB_HH] = instr_alu_export,
10775         [INSTR_ALU_SUB_MI] = instr_alu_export,
10776         [INSTR_ALU_SUB_HI] = instr_alu_export,
10777
10778         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10779         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10780         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10781         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10782
10783         [INSTR_ALU_AND] = instr_alu_export,
10784         [INSTR_ALU_AND_MH] = instr_alu_export,
10785         [INSTR_ALU_AND_HM] = instr_alu_export,
10786         [INSTR_ALU_AND_HH] = instr_alu_export,
10787         [INSTR_ALU_AND_I] = instr_alu_export,
10788
10789         [INSTR_ALU_OR] = instr_alu_export,
10790         [INSTR_ALU_OR_MH] = instr_alu_export,
10791         [INSTR_ALU_OR_HM] = instr_alu_export,
10792         [INSTR_ALU_OR_HH] = instr_alu_export,
10793         [INSTR_ALU_OR_I] = instr_alu_export,
10794
10795         [INSTR_ALU_XOR] = instr_alu_export,
10796         [INSTR_ALU_XOR_MH] = instr_alu_export,
10797         [INSTR_ALU_XOR_HM] = instr_alu_export,
10798         [INSTR_ALU_XOR_HH] = instr_alu_export,
10799         [INSTR_ALU_XOR_I] = instr_alu_export,
10800
10801         [INSTR_ALU_SHL] = instr_alu_export,
10802         [INSTR_ALU_SHL_MH] = instr_alu_export,
10803         [INSTR_ALU_SHL_HM] = instr_alu_export,
10804         [INSTR_ALU_SHL_HH] = instr_alu_export,
10805         [INSTR_ALU_SHL_MI] = instr_alu_export,
10806         [INSTR_ALU_SHL_HI] = instr_alu_export,
10807
10808         [INSTR_ALU_SHR] = instr_alu_export,
10809         [INSTR_ALU_SHR_MH] = instr_alu_export,
10810         [INSTR_ALU_SHR_HM] = instr_alu_export,
10811         [INSTR_ALU_SHR_HH] = instr_alu_export,
10812         [INSTR_ALU_SHR_MI] = instr_alu_export,
10813         [INSTR_ALU_SHR_HI] = instr_alu_export,
10814
10815         [INSTR_REGPREFETCH_RH] = instr_reg_export,
10816         [INSTR_REGPREFETCH_RM] = instr_reg_export,
10817         [INSTR_REGPREFETCH_RI] = instr_reg_export,
10818
10819         [INSTR_REGRD_HRH] = instr_reg_export,
10820         [INSTR_REGRD_HRM] = instr_reg_export,
10821         [INSTR_REGRD_MRH] = instr_reg_export,
10822         [INSTR_REGRD_MRM] = instr_reg_export,
10823         [INSTR_REGRD_HRI] = instr_reg_export,
10824         [INSTR_REGRD_MRI] = instr_reg_export,
10825
10826         [INSTR_REGWR_RHH] = instr_reg_export,
10827         [INSTR_REGWR_RHM] = instr_reg_export,
10828         [INSTR_REGWR_RMH] = instr_reg_export,
10829         [INSTR_REGWR_RMM] = instr_reg_export,
10830         [INSTR_REGWR_RHI] = instr_reg_export,
10831         [INSTR_REGWR_RMI] = instr_reg_export,
10832         [INSTR_REGWR_RIH] = instr_reg_export,
10833         [INSTR_REGWR_RIM] = instr_reg_export,
10834         [INSTR_REGWR_RII] = instr_reg_export,
10835
10836         [INSTR_REGADD_RHH] = instr_reg_export,
10837         [INSTR_REGADD_RHM] = instr_reg_export,
10838         [INSTR_REGADD_RMH] = instr_reg_export,
10839         [INSTR_REGADD_RMM] = instr_reg_export,
10840         [INSTR_REGADD_RHI] = instr_reg_export,
10841         [INSTR_REGADD_RMI] = instr_reg_export,
10842         [INSTR_REGADD_RIH] = instr_reg_export,
10843         [INSTR_REGADD_RIM] = instr_reg_export,
10844         [INSTR_REGADD_RII] = instr_reg_export,
10845
10846         [INSTR_METPREFETCH_H] = instr_meter_export,
10847         [INSTR_METPREFETCH_M] = instr_meter_export,
10848         [INSTR_METPREFETCH_I] = instr_meter_export,
10849
10850         [INSTR_METER_HHM] = instr_meter_export,
10851         [INSTR_METER_HHI] = instr_meter_export,
10852         [INSTR_METER_HMM] = instr_meter_export,
10853         [INSTR_METER_HMI] = instr_meter_export,
10854         [INSTR_METER_MHM] = instr_meter_export,
10855         [INSTR_METER_MHI] = instr_meter_export,
10856         [INSTR_METER_MMM] = instr_meter_export,
10857         [INSTR_METER_MMI] = instr_meter_export,
10858         [INSTR_METER_IHM] = instr_meter_export,
10859         [INSTR_METER_IHI] = instr_meter_export,
10860         [INSTR_METER_IMM] = instr_meter_export,
10861         [INSTR_METER_IMI] = instr_meter_export,
10862
10863         [INSTR_TABLE] = instr_table_export,
10864         [INSTR_TABLE_AF] = instr_table_export,
10865         [INSTR_SELECTOR] = instr_table_export,
10866         [INSTR_LEARNER] = instr_table_export,
10867         [INSTR_LEARNER_AF] = instr_table_export,
10868
10869         [INSTR_LEARNER_LEARN] = instr_learn_export,
10870         [INSTR_LEARNER_FORGET] = instr_forget_export,
10871
10872         [INSTR_EXTERN_OBJ] = instr_extern_export,
10873         [INSTR_EXTERN_FUNC] = instr_extern_export,
10874
10875         [INSTR_JMP] = instr_jmp_export,
10876         [INSTR_JMP_VALID] = instr_jmp_export,
10877         [INSTR_JMP_INVALID] = instr_jmp_export,
10878         [INSTR_JMP_HIT] = instr_jmp_export,
10879         [INSTR_JMP_MISS] = instr_jmp_export,
10880         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
10881         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
10882
10883         [INSTR_JMP_EQ] = instr_jmp_export,
10884         [INSTR_JMP_EQ_MH] = instr_jmp_export,
10885         [INSTR_JMP_EQ_HM] = instr_jmp_export,
10886         [INSTR_JMP_EQ_HH] = instr_jmp_export,
10887         [INSTR_JMP_EQ_I] = instr_jmp_export,
10888
10889         [INSTR_JMP_NEQ] = instr_jmp_export,
10890         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
10891         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
10892         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
10893         [INSTR_JMP_NEQ_I] = instr_jmp_export,
10894
10895         [INSTR_JMP_LT] = instr_jmp_export,
10896         [INSTR_JMP_LT_MH] = instr_jmp_export,
10897         [INSTR_JMP_LT_HM] = instr_jmp_export,
10898         [INSTR_JMP_LT_HH] = instr_jmp_export,
10899         [INSTR_JMP_LT_MI] = instr_jmp_export,
10900         [INSTR_JMP_LT_HI] = instr_jmp_export,
10901
10902         [INSTR_JMP_GT] = instr_jmp_export,
10903         [INSTR_JMP_GT_MH] = instr_jmp_export,
10904         [INSTR_JMP_GT_HM] = instr_jmp_export,
10905         [INSTR_JMP_GT_HH] = instr_jmp_export,
10906         [INSTR_JMP_GT_MI] = instr_jmp_export,
10907         [INSTR_JMP_GT_HI] = instr_jmp_export,
10908
10909         [INSTR_RETURN] = instr_return_export,
10910 };
10911
10912 static void
10913 action_data_codegen(struct action *a, FILE *f)
10914 {
10915         uint32_t i;
10916
10917         fprintf(f,
10918                 "static const struct instruction action_%s_instructions[] = {\n",
10919                 a->name);
10920
10921         for (i = 0; i < a->n_instructions; i++) {
10922                 struct instruction *instr = &a->instructions[i];
10923                 instruction_export_t func = export_table[instr->type];
10924
10925                 func(instr, f);
10926         }
10927
10928         fprintf(f, "};\n");
10929 }
10930
10931 static const char *
10932 instr_type_to_func(struct instruction *instr)
10933 {
10934         switch (instr->type) {
10935         case INSTR_RX: return NULL;
10936
10937         case INSTR_TX: return "__instr_tx_exec";
10938         case INSTR_TX_I: return "__instr_tx_i_exec";
10939
10940         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
10941         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
10942         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
10943         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
10944         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
10945         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
10946         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
10947         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
10948
10949         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
10950
10951         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
10952
10953         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
10954         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
10955         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
10956         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
10957         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
10958         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
10959         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
10960         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
10961         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
10962
10963         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
10964         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
10965
10966         case INSTR_MOV: return "__instr_mov_exec";
10967         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
10968         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
10969         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
10970         case INSTR_MOV_I: return "__instr_mov_i_exec";
10971
10972         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
10973         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
10974         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
10975         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
10976         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
10977         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
10978         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
10979         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
10980
10981         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
10982         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
10983         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
10984         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
10985         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
10986         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
10987
10988         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
10989         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
10990         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
10991         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
10992         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
10993         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
10994
10995         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
10996         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
10997         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
10998         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
10999
11000         case INSTR_ALU_AND: return "__instr_alu_and_exec";
11001         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11002         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11003         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11004         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11005
11006         case INSTR_ALU_OR: return "__instr_alu_or_exec";
11007         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11008         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11009         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11010         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11011
11012         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11013         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11014         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11015         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11016         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11017
11018         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11019         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11020         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11021         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11022         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11023         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11024
11025         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11026         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11027         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11028         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11029         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11030         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11031
11032         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11033         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11034         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11035
11036         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11037         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11038         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11039         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11040         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11041         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11042
11043         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11044         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11045         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11046         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11047         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11048         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11049         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11050         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11051         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11052
11053         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11054         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11055         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11056         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11057         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11058         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11059         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11060         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11061         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11062
11063         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11064         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11065         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11066
11067         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11068         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11069         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11070         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11071         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11072         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11073         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11074         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11075         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11076         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11077         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11078         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11079
11080         case INSTR_TABLE: return NULL;
11081         case INSTR_TABLE_AF: return NULL;
11082         case INSTR_SELECTOR: return NULL;
11083         case INSTR_LEARNER: return NULL;
11084         case INSTR_LEARNER_AF: return NULL;
11085
11086         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11087         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11088
11089         case INSTR_EXTERN_OBJ: return NULL;
11090         case INSTR_EXTERN_FUNC: return NULL;
11091
11092         case INSTR_JMP: return NULL;
11093         case INSTR_JMP_VALID: return NULL;
11094         case INSTR_JMP_INVALID: return NULL;
11095         case INSTR_JMP_HIT: return NULL;
11096         case INSTR_JMP_MISS: return NULL;
11097         case INSTR_JMP_ACTION_HIT: return NULL;
11098         case INSTR_JMP_ACTION_MISS: return NULL;
11099         case INSTR_JMP_EQ: return NULL;
11100         case INSTR_JMP_EQ_MH: return NULL;
11101         case INSTR_JMP_EQ_HM: return NULL;
11102         case INSTR_JMP_EQ_HH: return NULL;
11103         case INSTR_JMP_EQ_I: return NULL;
11104         case INSTR_JMP_NEQ: return NULL;
11105         case INSTR_JMP_NEQ_MH: return NULL;
11106         case INSTR_JMP_NEQ_HM: return NULL;
11107         case INSTR_JMP_NEQ_HH: return NULL;
11108         case INSTR_JMP_NEQ_I: return NULL;
11109         case INSTR_JMP_LT: return NULL;
11110         case INSTR_JMP_LT_MH: return NULL;
11111         case INSTR_JMP_LT_HM: return NULL;
11112         case INSTR_JMP_LT_HH: return NULL;
11113         case INSTR_JMP_LT_MI: return NULL;
11114         case INSTR_JMP_LT_HI: return NULL;
11115         case INSTR_JMP_GT: return NULL;
11116         case INSTR_JMP_GT_MH: return NULL;
11117         case INSTR_JMP_GT_HM: return NULL;
11118         case INSTR_JMP_GT_HH: return NULL;
11119         case INSTR_JMP_GT_MI: return NULL;
11120         case INSTR_JMP_GT_HI: return NULL;
11121
11122         case INSTR_RETURN: return NULL;
11123
11124         default: return NULL;
11125         }
11126 }
11127
11128 static void
11129 action_instr_does_tx_codegen(struct action *a,
11130                         uint32_t instr_pos,
11131                         struct instruction *instr,
11132                         FILE *f)
11133 {
11134         fprintf(f,
11135                 "%s(p, t, &action_%s_instructions[%u]);\n"
11136                 "\tthread_ip_reset(p, t);\n"
11137                 "\tinstr_rx_exec(p);\n"
11138                 "\treturn;\n",
11139                 instr_type_to_func(instr),
11140                 a->name,
11141                 instr_pos);
11142 }
11143
11144 static void
11145 action_instr_extern_obj_codegen(struct action *a,
11146                                 uint32_t instr_pos,
11147                                 FILE *f)
11148 {
11149         fprintf(f,
11150                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11151                 a->name,
11152                 instr_pos);
11153 }
11154
11155 static void
11156 action_instr_extern_func_codegen(struct action *a,
11157                                  uint32_t instr_pos,
11158                                  FILE *f)
11159 {
11160         fprintf(f,
11161                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11162                 a->name,
11163                 instr_pos);
11164 }
11165
11166 static void
11167 action_instr_jmp_codegen(struct action *a,
11168                          uint32_t instr_pos,
11169                          struct instruction *instr,
11170                          struct instruction_data *data,
11171                          FILE *f)
11172 {
11173         switch (instr->type) {
11174         case INSTR_JMP:
11175                 fprintf(f,
11176                         "goto %s;\n",
11177                         data->jmp_label);
11178                 return;
11179
11180         case INSTR_JMP_VALID:
11181                 fprintf(f,
11182                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11183                         "\t\tgoto %s;\n",
11184                         a->name,
11185                         instr_pos,
11186                         data->jmp_label);
11187                 return;
11188
11189         case INSTR_JMP_INVALID:
11190                 fprintf(f,
11191                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11192                         "\t\tgoto %s;\n",
11193                         a->name,
11194                         instr_pos,
11195                         data->jmp_label);
11196                 return;
11197
11198         case INSTR_JMP_HIT:
11199                 fprintf(f,
11200                         "if (t->hit)\n"
11201                         "\t\tgoto %s;\n",
11202                         data->jmp_label);
11203                 return;
11204
11205         case INSTR_JMP_MISS:
11206                 fprintf(f,
11207                         "if (!t->hit)\n"
11208                         "\t\tgoto %s;\n",
11209                         data->jmp_label);
11210                 return;
11211
11212         case INSTR_JMP_ACTION_HIT:
11213                 fprintf(f,
11214                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11215                         "\t\tgoto %s;\n",
11216                         a->name,
11217                         instr_pos,
11218                         data->jmp_label);
11219                 return;
11220
11221         case INSTR_JMP_ACTION_MISS:
11222                 fprintf(f,
11223                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11224                         "\t\tgoto %s;\n",
11225                         a->name,
11226                         instr_pos,
11227                         data->jmp_label);
11228                 return;
11229
11230         case INSTR_JMP_EQ:
11231                 fprintf(f,
11232                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11233                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11234                         "\t\tgoto %s;\n",
11235                         a->name,
11236                         instr_pos,
11237                         a->name,
11238                         instr_pos,
11239                         data->jmp_label);
11240                 return;
11241
11242         case INSTR_JMP_EQ_MH:
11243                 fprintf(f,
11244                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11245                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11246                         "\t\tgoto %s;\n",
11247                         a->name,
11248                         instr_pos,
11249                         a->name,
11250                         instr_pos,
11251                         data->jmp_label);
11252                 return;
11253
11254         case INSTR_JMP_EQ_HM:
11255                 fprintf(f,
11256                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11257                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11258                         "\t\tgoto %s;\n",
11259                         a->name,
11260                         instr_pos,
11261                         a->name,
11262                         instr_pos,
11263                         data->jmp_label);
11264                 return;
11265
11266         case INSTR_JMP_EQ_HH:
11267                 fprintf(f,
11268                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11269                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11270                         "\t\tgoto %s;\n",
11271                         a->name,
11272                         instr_pos,
11273                         a->name,
11274                         instr_pos,
11275                         data->jmp_label);
11276                 return;
11277
11278         case INSTR_JMP_EQ_I:
11279                 fprintf(f,
11280                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11281                         "action_%s_instructions[%u].jmp.b_val)\n"
11282                         "\t\tgoto %s;\n",
11283                         a->name,
11284                         instr_pos,
11285                         a->name,
11286                         instr_pos,
11287                         data->jmp_label);
11288                 return;
11289
11290         case INSTR_JMP_NEQ:
11291                 fprintf(f,
11292                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11293                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11294                         "\t\tgoto %s;\n",
11295                         a->name,
11296                         instr_pos,
11297                         a->name,
11298                         instr_pos,
11299                         data->jmp_label);
11300                 return;
11301
11302         case INSTR_JMP_NEQ_MH:
11303                 fprintf(f,
11304                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11305                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11306                         "\t\tgoto %s;\n",
11307                         a->name,
11308                         instr_pos,
11309                         a->name,
11310                         instr_pos,
11311                         data->jmp_label);
11312                 return;
11313
11314         case INSTR_JMP_NEQ_HM:
11315                 fprintf(f,
11316                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11317                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11318                         "\t\tgoto %s;\n",
11319                         a->name,
11320                         instr_pos,
11321                         a->name,
11322                         instr_pos,
11323                         data->jmp_label);
11324                 return;
11325
11326         case INSTR_JMP_NEQ_HH:
11327                 fprintf(f,
11328                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11329                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11330                         "\t\tgoto %s;\n",
11331                         a->name,
11332                         instr_pos,
11333                         a->name,
11334                         instr_pos,
11335                         data->jmp_label);
11336                 return;
11337
11338         case INSTR_JMP_NEQ_I:
11339                 fprintf(f,
11340                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11341                         "action_%s_instructions[%u].jmp.b_val)\n"
11342                         "\t\tgoto %s;\n",
11343                         a->name,
11344                         instr_pos,
11345                         a->name,
11346                         instr_pos,
11347                         data->jmp_label);
11348                 return;
11349
11350         case INSTR_JMP_LT:
11351                 fprintf(f,
11352                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11353                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11354                         "\t\tgoto %s;\n",
11355                         a->name,
11356                         instr_pos,
11357                         a->name,
11358                         instr_pos,
11359                         data->jmp_label);
11360                 return;
11361
11362         case INSTR_JMP_LT_MH:
11363                 fprintf(f,
11364                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11365                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11366                         "\t\tgoto %s;\n",
11367                         a->name,
11368                         instr_pos,
11369                         a->name,
11370                         instr_pos,
11371                         data->jmp_label);
11372                 return;
11373
11374         case INSTR_JMP_LT_HM:
11375                 fprintf(f,
11376                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11377                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11378                         "\t\tgoto %s;\n",
11379                         a->name,
11380                         instr_pos,
11381                         a->name,
11382                         instr_pos,
11383                         data->jmp_label);
11384                 return;
11385
11386         case INSTR_JMP_LT_HH:
11387                 fprintf(f,
11388                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11389                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11390                         "\t\tgoto %s;\n",
11391                         a->name,
11392                         instr_pos,
11393                         a->name,
11394                         instr_pos,
11395                         data->jmp_label);
11396                 return;
11397
11398         case INSTR_JMP_LT_MI:
11399                 fprintf(f,
11400                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11401                         "action_%s_instructions[%u].jmp.b_val)\n"
11402                         "\t\tgoto %s;\n",
11403                         a->name,
11404                         instr_pos,
11405                         a->name,
11406                         instr_pos,
11407                         data->jmp_label);
11408                 return;
11409
11410         case INSTR_JMP_LT_HI:
11411                 fprintf(f,
11412                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11413                         "action_%s_instructions[%u].jmp.b_val)\n"
11414                         "\t\tgoto %s;\n",
11415                         a->name,
11416                         instr_pos,
11417                         a->name,
11418                         instr_pos,
11419                         data->jmp_label);
11420                 return;
11421
11422         case INSTR_JMP_GT:
11423                 fprintf(f,
11424                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11425                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11426                         "\t\tgoto %s;\n",
11427                         a->name,
11428                         instr_pos,
11429                         a->name,
11430                         instr_pos,
11431                         data->jmp_label);
11432                 return;
11433
11434         case INSTR_JMP_GT_MH:
11435                 fprintf(f,
11436                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11437                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11438                         "\t\tgoto %s;\n",
11439                         a->name,
11440                         instr_pos,
11441                         a->name,
11442                         instr_pos,
11443                         data->jmp_label);
11444                 return;
11445
11446         case INSTR_JMP_GT_HM:
11447                 fprintf(f,
11448                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11449                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11450                         "\t\tgoto %s;\n",
11451                         a->name,
11452                         instr_pos,
11453                         a->name,
11454                         instr_pos,
11455                         data->jmp_label);
11456                 return;
11457
11458         case INSTR_JMP_GT_HH:
11459                 fprintf(f,
11460                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11461                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11462                         "\t\tgoto %s;\n",
11463                         a->name,
11464                         instr_pos,
11465                         a->name,
11466                         instr_pos,
11467                         data->jmp_label);
11468                 return;
11469
11470         case INSTR_JMP_GT_MI:
11471                 fprintf(f,
11472                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11473                         "action_%s_instructions[%u].jmp.b_val)\n"
11474                         "\t\tgoto %s;\n",
11475                         a->name,
11476                         instr_pos,
11477                         a->name,
11478                         instr_pos,
11479                         data->jmp_label);
11480                 return;
11481
11482         case INSTR_JMP_GT_HI:
11483                 fprintf(f,
11484                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11485                         "action_%s_instructions[%u].jmp.b_val)\n"
11486                         "\t\tgoto %s;\n",
11487                         a->name,
11488                         instr_pos,
11489                         a->name,
11490                         instr_pos,
11491                         data->jmp_label);
11492                 return;
11493
11494         default:
11495                 return;
11496         }
11497 }
11498
11499 static void
11500 action_instr_return_codegen(FILE *f)
11501 {
11502         fprintf(f,
11503                 "return;\n");
11504 }
11505
11506 static void
11507 action_instr_codegen(struct action *a, FILE *f)
11508 {
11509         uint32_t i;
11510
11511         fprintf(f,
11512                 "void\n"
11513                 "action_%s_run(struct rte_swx_pipeline *p)\n"
11514                 "{\n"
11515                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11516                 "\n",
11517                 a->name);
11518
11519         for (i = 0; i < a->n_instructions; i++) {
11520                 struct instruction *instr = &a->instructions[i];
11521                 struct instruction_data *data = &a->instruction_data[i];
11522
11523                 /* Label, if present. */
11524                 if (data->label[0])
11525                         fprintf(f, "\n%s : ", data->label);
11526                 else
11527                         fprintf(f, "\n\t");
11528
11529                 /* TX instruction type. */
11530                 if (instruction_does_tx(instr)) {
11531                         action_instr_does_tx_codegen(a, i, instr, f);
11532                         continue;
11533                 }
11534
11535                 /* Extern object/function instruction type. */
11536                 if (instr->type == INSTR_EXTERN_OBJ) {
11537                         action_instr_extern_obj_codegen(a, i, f);
11538                         continue;
11539                 }
11540
11541                 if (instr->type == INSTR_EXTERN_FUNC) {
11542                         action_instr_extern_func_codegen(a, i, f);
11543                         continue;
11544                 }
11545
11546                 /* Jump instruction type. */
11547                 if (instruction_is_jmp(instr)) {
11548                         action_instr_jmp_codegen(a, i, instr, data, f);
11549                         continue;
11550                 }
11551
11552                 /* Return instruction type. */
11553                 if (instr->type == INSTR_RETURN) {
11554                         action_instr_return_codegen(f);
11555                         continue;
11556                 }
11557
11558                 /* Any other instruction type. */
11559                 fprintf(f,
11560                         "%s(p, t, &action_%s_instructions[%u]);\n",
11561                         instr_type_to_func(instr),
11562                         a->name,
11563                         i);
11564         }
11565
11566         fprintf(f, "}\n\n");
11567 }
11568
11569 struct instruction_group {
11570         TAILQ_ENTRY(instruction_group) node;
11571
11572         uint32_t group_id;
11573
11574         uint32_t first_instr_id;
11575
11576         uint32_t last_instr_id;
11577
11578         instr_exec_t func;
11579 };
11580
11581 TAILQ_HEAD(instruction_group_list, instruction_group);
11582
11583 static struct instruction_group *
11584 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11585 {
11586         struct instruction_group *g;
11587
11588         TAILQ_FOREACH(g, igl, node)
11589                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11590                         return g;
11591
11592         return NULL;
11593 }
11594
11595 static void
11596 instruction_group_list_free(struct instruction_group_list *igl)
11597 {
11598         if (!igl)
11599                 return;
11600
11601         for ( ; ; ) {
11602                 struct instruction_group *g;
11603
11604                 g = TAILQ_FIRST(igl);
11605                 if (!g)
11606                         break;
11607
11608                 TAILQ_REMOVE(igl, g, node);
11609                 free(g);
11610         }
11611
11612         free(igl);
11613 }
11614
11615 static struct instruction_group_list *
11616 instruction_group_list_create(struct rte_swx_pipeline *p)
11617 {
11618         struct instruction_group_list *igl = NULL;
11619         struct instruction_group *g = NULL;
11620         uint32_t n_groups = 0, i;
11621
11622         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11623                 goto error;
11624
11625         /* List init. */
11626         igl = calloc(1, sizeof(struct instruction_group_list));
11627         if (!igl)
11628                 goto error;
11629
11630         TAILQ_INIT(igl);
11631
11632         /* Allocate the first group. */
11633         g = calloc(1, sizeof(struct instruction_group));
11634         if (!g)
11635                 goto error;
11636
11637         /* Iteration 1: Separate the instructions into groups based on the thread yield
11638          * instructions. Do not worry about the jump instructions at this point.
11639          */
11640         for (i = 0; i < p->n_instructions; i++) {
11641                 struct instruction *instr = &p->instructions[i];
11642
11643                 /* Check for thread yield instructions. */
11644                 if (!instruction_does_thread_yield(instr))
11645                         continue;
11646
11647                 /* If the current group contains at least one instruction, then finalize it (with
11648                  * the previous instruction), add it to the list and allocate a new group (that
11649                  * starts with the current instruction).
11650                  */
11651                 if (i - g->first_instr_id) {
11652                         /* Finalize the group. */
11653                         g->last_instr_id = i - 1;
11654
11655                         /* Add the group to the list. Advance the number of groups. */
11656                         TAILQ_INSERT_TAIL(igl, g, node);
11657                         n_groups++;
11658
11659                         /* Allocate a new group. */
11660                         g = calloc(1, sizeof(struct instruction_group));
11661                         if (!g)
11662                                 goto error;
11663
11664                         /* Initialize the new group. */
11665                         g->group_id = n_groups;
11666                         g->first_instr_id = i;
11667                 }
11668
11669                 /* Finalize the current group (with the current instruction, therefore this group
11670                  * contains just the current thread yield instruction), add it to the list and
11671                  * allocate a new group (that starts with the next instruction).
11672                  */
11673
11674                 /* Finalize the group. */
11675                 g->last_instr_id = i;
11676
11677                 /* Add the group to the list. Advance the number of groups. */
11678                 TAILQ_INSERT_TAIL(igl, g, node);
11679                 n_groups++;
11680
11681                 /* Allocate a new group. */
11682                 g = calloc(1, sizeof(struct instruction_group));
11683                 if (!g)
11684                         goto error;
11685
11686                 /* Initialize the new group. */
11687                 g->group_id = n_groups;
11688                 g->first_instr_id = i + 1;
11689         }
11690
11691         /* Handle the last group. */
11692         if (i - g->first_instr_id) {
11693                 /* Finalize the group. */
11694                 g->last_instr_id = i - 1;
11695
11696                 /* Add the group to the list. Advance the number of groups. */
11697                 TAILQ_INSERT_TAIL(igl, g, node);
11698                 n_groups++;
11699         } else
11700                 free(g);
11701
11702         g = NULL;
11703
11704         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11705          * the destination of a jump instruction located in a different group ("far jump"), then the
11706          * current group has to be split, so that the instruction representing the far jump
11707          * destination is at the start of its group.
11708          */
11709         for ( ; ; ) {
11710                 int is_modified = 0;
11711
11712                 for (i = 0; i < p->n_instructions; i++) {
11713                         struct instruction_data *data = &p->instruction_data[i];
11714                         struct instruction_group *g;
11715                         uint32_t j;
11716
11717                         /* Continue when the current instruction is not a jump destination. */
11718                         if (!data->n_users)
11719                                 continue;
11720
11721                         g = instruction_group_list_group_find(igl, i);
11722                         if (!g)
11723                                 goto error;
11724
11725                         /* Find out all the jump instructions with this destination. */
11726                         for (j = 0; j < p->n_instructions; j++) {
11727                                 struct instruction *jmp_instr = &p->instructions[j];
11728                                 struct instruction_data *jmp_data = &p->instruction_data[j];
11729                                 struct instruction_group *jmp_g, *new_g;
11730
11731                                 /* Continue when not a jump instruction. Even when jump instruction,
11732                                  * continue when the jump destination is not this instruction.
11733                                  */
11734                                 if (!instruction_is_jmp(jmp_instr) ||
11735                                     strcmp(jmp_data->jmp_label, data->label))
11736                                         continue;
11737
11738                                 jmp_g = instruction_group_list_group_find(igl, j);
11739                                 if (!jmp_g)
11740                                         goto error;
11741
11742                                 /* Continue when both the jump instruction and the jump destination
11743                                  * instruction are in the same group. Even when in different groups,
11744                                  * still continue if the jump destination instruction is already the
11745                                  * first instruction of its group.
11746                                  */
11747                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11748                                         continue;
11749
11750                                 /* Split the group of the current jump destination instruction to
11751                                  * make this instruction the first instruction of a new group.
11752                                  */
11753                                 new_g = calloc(1, sizeof(struct instruction_group));
11754                                 if (!new_g)
11755                                         goto error;
11756
11757                                 new_g->group_id = n_groups;
11758                                 new_g->first_instr_id = i;
11759                                 new_g->last_instr_id = g->last_instr_id;
11760
11761                                 g->last_instr_id = i - 1;
11762
11763                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11764                                 n_groups++;
11765                                 is_modified = 1;
11766
11767                                 /* The decision to split this group (to make the current instruction
11768                                  * the first instruction of a new group) is already taken and fully
11769                                  * implemented, so no need to search for more reasons to do it.
11770                                  */
11771                                 break;
11772                         }
11773                 }
11774
11775                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11776                  * previously considered local (i.e. the jump destination is in the same group as
11777                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11778                  * different group than the jump instruction). Wost case scenario: each instruction
11779                  * that is a jump destination ends up as the first instruction of its group.
11780                  */
11781                 if (!is_modified)
11782                         break;
11783         }
11784
11785         /* Re-assign the group IDs to be in incremental order. */
11786         i = 0;
11787         TAILQ_FOREACH(g, igl, node) {
11788                 g->group_id = i;
11789
11790                 i++;
11791         }
11792
11793         return igl;
11794
11795 error:
11796         instruction_group_list_free(igl);
11797
11798         free(g);
11799
11800         return NULL;
11801 }
11802
11803 static void
11804 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11805                                uint32_t instr_pos,
11806                                struct instruction *instr,
11807                                FILE *f)
11808 {
11809         fprintf(f,
11810                 "%s(p, t, &pipeline_instructions[%u]);\n"
11811                 "\tthread_ip_reset(p, t);\n"
11812                 "\tinstr_rx_exec(p);\n"
11813                 "\treturn;\n",
11814                 instr_type_to_func(instr),
11815                 instr_pos);
11816 }
11817
11818 static int
11819 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11820                            struct instruction_group_list *igl,
11821                            uint32_t jmp_instr_id,
11822                            struct instruction *jmp_instr,
11823                            struct instruction_data *jmp_data,
11824                            FILE *f)
11825 {
11826         struct instruction_group *jmp_g, *g;
11827         struct instruction_data *data;
11828         uint32_t instr_id;
11829
11830         switch (jmp_instr->type) {
11831         case INSTR_JMP:
11832                 break;
11833
11834         case INSTR_JMP_VALID:
11835                 fprintf(f,
11836                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11837                         jmp_instr_id);
11838                 break;
11839
11840         case INSTR_JMP_INVALID:
11841                 fprintf(f,
11842                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11843                         jmp_instr_id);
11844                 break;
11845
11846         case INSTR_JMP_HIT:
11847                 fprintf(f,
11848                         "if (t->hit)\n");
11849                 break;
11850
11851         case INSTR_JMP_MISS:
11852                 fprintf(f,
11853                         "if (!t->hit)\n");
11854                 break;
11855
11856         case INSTR_JMP_ACTION_HIT:
11857                 fprintf(f,
11858                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
11859                         jmp_instr_id);
11860                 break;
11861
11862         case INSTR_JMP_ACTION_MISS:
11863                 fprintf(f,
11864                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
11865                         jmp_instr_id);
11866                 break;
11867
11868         case INSTR_JMP_EQ:
11869                 fprintf(f,
11870                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11871                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11872                         jmp_instr_id,
11873                         jmp_instr_id);
11874                 break;
11875
11876         case INSTR_JMP_EQ_MH:
11877                 fprintf(f,
11878                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11879                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11880                         jmp_instr_id,
11881                         jmp_instr_id);
11882                 break;
11883
11884         case INSTR_JMP_EQ_HM:
11885                 fprintf(f,
11886                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11887                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11888                         jmp_instr_id,
11889                         jmp_instr_id);
11890                 break;
11891
11892         case INSTR_JMP_EQ_HH:
11893                 fprintf(f,
11894                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11895                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11896                         jmp_instr_id,
11897                         jmp_instr_id);
11898                 break;
11899
11900         case INSTR_JMP_EQ_I:
11901                 fprintf(f,
11902                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11903                         "pipeline_instructions[%u].jmp.b_val)",
11904                         jmp_instr_id,
11905                         jmp_instr_id);
11906                 break;
11907
11908         case INSTR_JMP_NEQ:
11909                 fprintf(f,
11910                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11911                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11912                         jmp_instr_id,
11913                         jmp_instr_id);
11914                 break;
11915
11916         case INSTR_JMP_NEQ_MH:
11917                 fprintf(f,
11918                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11919                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11920                         jmp_instr_id,
11921                         jmp_instr_id);
11922                 break;
11923
11924         case INSTR_JMP_NEQ_HM:
11925                 fprintf(f,
11926                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11927                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11928                         jmp_instr_id,
11929                         jmp_instr_id);
11930                 break;
11931
11932         case INSTR_JMP_NEQ_HH:
11933                 fprintf(f,
11934                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11935                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11936                         jmp_instr_id,
11937                         jmp_instr_id);
11938                 break;
11939
11940         case INSTR_JMP_NEQ_I:
11941                 fprintf(f,
11942                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11943                         "pipeline_instructions[%u].jmp.b_val)",
11944                         jmp_instr_id,
11945                         jmp_instr_id);
11946                 break;
11947
11948         case INSTR_JMP_LT:
11949                 fprintf(f,
11950                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11951                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11952                         jmp_instr_id,
11953                         jmp_instr_id);
11954                 break;
11955
11956         case INSTR_JMP_LT_MH:
11957                 fprintf(f,
11958                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11959                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11960                         jmp_instr_id,
11961                         jmp_instr_id);
11962                 break;
11963
11964         case INSTR_JMP_LT_HM:
11965                 fprintf(f,
11966                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11967                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11968                         jmp_instr_id,
11969                         jmp_instr_id);
11970                 break;
11971
11972         case INSTR_JMP_LT_HH:
11973                 fprintf(f,
11974                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11975                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11976                         jmp_instr_id,
11977                         jmp_instr_id);
11978                 break;
11979
11980         case INSTR_JMP_LT_MI:
11981                 fprintf(f,
11982                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11983                         "pipeline_instructions[%u].jmp.b_val)",
11984                         jmp_instr_id,
11985                         jmp_instr_id);
11986                 break;
11987
11988         case INSTR_JMP_LT_HI:
11989                 fprintf(f,
11990                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11991                         "pipeline_instructions[%u].jmp.b_val)",
11992                         jmp_instr_id,
11993                         jmp_instr_id);
11994                 break;
11995
11996         case INSTR_JMP_GT:
11997                 fprintf(f,
11998                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
11999                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12000                         jmp_instr_id,
12001                         jmp_instr_id);
12002                 break;
12003
12004         case INSTR_JMP_GT_MH:
12005                 fprintf(f,
12006                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12007                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12008                         jmp_instr_id,
12009                         jmp_instr_id);
12010                 break;
12011
12012         case INSTR_JMP_GT_HM:
12013                 fprintf(f,
12014                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12015                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12016                         jmp_instr_id,
12017                         jmp_instr_id);
12018                 break;
12019
12020         case INSTR_JMP_GT_HH:
12021                 fprintf(f,
12022                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12023                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12024                         jmp_instr_id,
12025                         jmp_instr_id);
12026                 break;
12027
12028         case INSTR_JMP_GT_MI:
12029                 fprintf(f,
12030                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12031                         "pipeline_instructions[%u].jmp.b_val)",
12032                         jmp_instr_id,
12033                         jmp_instr_id);
12034                 break;
12035
12036         case INSTR_JMP_GT_HI:
12037                 fprintf(f,
12038                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12039                         "pipeline_instructions[%u].jmp.b_val)",
12040                         jmp_instr_id,
12041                         jmp_instr_id);
12042                 break;
12043
12044         default:
12045                 break;
12046         }
12047
12048         /* Find the instruction group of the jump instruction. */
12049         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12050         if (!jmp_g)
12051                 return -EINVAL;
12052
12053         /* Find the instruction group of the jump destination instruction. */
12054         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12055         if (!data)
12056                 return -EINVAL;
12057
12058         instr_id = data - p->instruction_data;
12059
12060         g = instruction_group_list_group_find(igl, instr_id);
12061         if (!g)
12062                 return -EINVAL;
12063
12064         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12065          * instruction group).
12066          */
12067         if (g->group_id == jmp_g->group_id)
12068                 fprintf(f,
12069                         "\n\t\tgoto %s;\n",
12070                         jmp_data->jmp_label);
12071         else
12072                 fprintf(f,
12073                         " {\n"
12074                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12075                         "\t\treturn;\n"
12076                         "\t}\n\n",
12077                         g->group_id);
12078
12079         return 0;
12080 }
12081
12082 static void
12083 instruction_group_list_codegen(struct instruction_group_list *igl,
12084                                struct rte_swx_pipeline *p,
12085                                FILE *f)
12086 {
12087         struct instruction_group *g;
12088         uint32_t i;
12089         int is_required = 0;
12090
12091         /* Check if code generation is required. */
12092         TAILQ_FOREACH(g, igl, node)
12093                 if (g->first_instr_id < g->last_instr_id)
12094                         is_required = 1;
12095
12096         if (!is_required)
12097                 return;
12098
12099         /* Generate the code for the pipeline instruction array. */
12100         fprintf(f,
12101                 "static const struct instruction pipeline_instructions[] = {\n");
12102
12103         for (i = 0; i < p->n_instructions; i++) {
12104                 struct instruction *instr = &p->instructions[i];
12105                 instruction_export_t func = export_table[instr->type];
12106
12107                 func(instr, f);
12108         }
12109
12110         fprintf(f, "};\n\n");
12111
12112         /* Generate the code for the pipeline functions: one function for each instruction group
12113          * that contains more than one instruction.
12114          */
12115         TAILQ_FOREACH(g, igl, node) {
12116                 struct instruction *last_instr;
12117                 uint32_t j;
12118
12119                 /* Skip if group contains a single instruction. */
12120                 if (g->last_instr_id == g->first_instr_id)
12121                         continue;
12122
12123                 /* Generate new pipeline function. */
12124                 fprintf(f,
12125                         "void\n"
12126                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12127                         "{\n"
12128                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12129                         "\n",
12130                         g->group_id);
12131
12132                 /* Generate the code for each pipeline instruction. */
12133                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12134                         struct instruction *instr = &p->instructions[j];
12135                         struct instruction_data *data = &p->instruction_data[j];
12136
12137                         /* Label, if present. */
12138                         if (data->label[0])
12139                                 fprintf(f, "\n%s : ", data->label);
12140                         else
12141                                 fprintf(f, "\n\t");
12142
12143                         /* TX instruction type. */
12144                         if (instruction_does_tx(instr)) {
12145                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12146                                 continue;
12147                         }
12148
12149                         /* Jump instruction type. */
12150                         if (instruction_is_jmp(instr)) {
12151                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12152                                 continue;
12153                         }
12154
12155                         /* Any other instruction type. */
12156                         fprintf(f,
12157                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12158                                 instr_type_to_func(instr),
12159                                 j);
12160                 }
12161
12162                 /* Finalize the generated pipeline function. For some instructions such as TX,
12163                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12164                  * decided unconditionally and the instruction pointer of the current thread set
12165                  * accordingly; for all the other instructions, the instruction pointer must be
12166                  * incremented now.
12167                  */
12168                 last_instr = &p->instructions[g->last_instr_id];
12169
12170                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12171                         fprintf(f,
12172                                 "thread_ip_inc(p);\n");
12173
12174                 fprintf(f,
12175                         "}\n"
12176                         "\n");
12177         }
12178 }
12179
12180 static uint32_t
12181 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12182 {
12183         struct instruction_group *g;
12184         uint32_t n_custom_instr = 0;
12185
12186         /* Groups with a single instruction: no function is generated for this group, the group
12187          * keeps its current instruction. Groups with more than two instructions: one function and
12188          * the associated custom instruction get generated for each such group.
12189          */
12190         TAILQ_FOREACH(g, igl, node) {
12191                 if (g->first_instr_id == g->last_instr_id)
12192                         continue;
12193
12194                 n_custom_instr++;
12195         }
12196
12197         return n_custom_instr;
12198 }
12199
12200 static int
12201 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12202 {
12203         struct action *a;
12204         FILE *f = NULL;
12205
12206         /* Create the .c file. */
12207         f = fopen("/tmp/pipeline.c", "w");
12208         if (!f)
12209                 return -EIO;
12210
12211         /* Include the .h file. */
12212         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12213
12214         /* Add the code for each action. */
12215         TAILQ_FOREACH(a, &p->actions, node) {
12216                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12217
12218                 action_data_codegen(a, f);
12219
12220                 fprintf(f, "\n");
12221
12222                 action_instr_codegen(a, f);
12223
12224                 fprintf(f, "\n");
12225         }
12226
12227         /* Add the pipeline code. */
12228         instruction_group_list_codegen(igl, p, f);
12229
12230         /* Close the .c file. */
12231         fclose(f);
12232
12233         return 0;
12234 }
12235
12236 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12237 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12238 #endif
12239
12240 static int
12241 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12242 {
12243         struct action *a;
12244         struct instruction_group *g;
12245         char *dir_in, *buffer = NULL;
12246         const char *dir_out;
12247         int status = 0;
12248
12249         /* Get the environment variables. */
12250         dir_in = getenv("RTE_INSTALL_DIR");
12251         if (!dir_in) {
12252                 status = -EINVAL;
12253                 goto free;
12254         }
12255
12256         dir_out = "/tmp";
12257
12258         /* Memory allocation for the command buffer. */
12259         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12260         if (!buffer) {
12261                 status = -ENOMEM;
12262                 goto free;
12263         }
12264
12265         snprintf(buffer,
12266                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12267                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12268                  "-I %s/lib/pipeline "
12269                  "-I %s/lib/eal/include "
12270                  "-I %s/lib/eal/x86/include "
12271                  "-I %s/lib/eal/include/generic "
12272                  "-I %s/lib/meter "
12273                  "-I %s/lib/port "
12274                  "-I %s/lib/table "
12275                  "-I %s/lib/pipeline "
12276                  "-I %s/config "
12277                  "-I %s/build "
12278                  "-I %s/lib/eal/linux/include "
12279                  ">%s/pipeline.log 2>&1 "
12280                  "&& "
12281                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12282                  ">>%s/pipeline.log 2>&1",
12283                  dir_out,
12284                  dir_out,
12285                  dir_in,
12286                  dir_in,
12287                  dir_in,
12288                  dir_in,
12289                  dir_in,
12290                  dir_in,
12291                  dir_in,
12292                  dir_in,
12293                  dir_in,
12294                  dir_in,
12295                  dir_in,
12296                  dir_out,
12297                  dir_out,
12298                  dir_out,
12299                  dir_out);
12300
12301         /* Build the shared object library. */
12302         status = system(buffer);
12303         if (status)
12304                 goto free;
12305
12306         /* Open library. */
12307         snprintf(buffer,
12308                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12309                  "%s/libpipeline.so",
12310                  dir_out);
12311
12312         p->lib = dlopen(buffer, RTLD_LAZY);
12313         if (!p->lib) {
12314                 status = -EIO;
12315                 goto free;
12316         }
12317
12318         /* Get the action function symbols. */
12319         TAILQ_FOREACH(a, &p->actions, node) {
12320                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12321
12322                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12323                 if (!p->action_funcs[a->id]) {
12324                         status = -EINVAL;
12325                         goto free;
12326                 }
12327         }
12328
12329         /* Get the pipeline function symbols. */
12330         TAILQ_FOREACH(g, igl, node) {
12331                 if (g->first_instr_id == g->last_instr_id)
12332                         continue;
12333
12334                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12335
12336                 g->func = dlsym(p->lib, buffer);
12337                 if (!g->func) {
12338                         status = -EINVAL;
12339                         goto free;
12340                 }
12341         }
12342
12343 free:
12344         if (status && p->lib) {
12345                 dlclose(p->lib);
12346                 p->lib = NULL;
12347         }
12348
12349         free(buffer);
12350
12351         return status;
12352 }
12353
12354 static int
12355 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12356                       struct instruction_group_list *igl)
12357 {
12358         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12359
12360         /* Check that enough space is available within the pipeline instruction table to store all
12361          * the custom instructions.
12362          */
12363         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12364                 return -ENOSPC;
12365
12366         return 0;
12367 }
12368
12369 static void
12370 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12371 {
12372         struct instruction_group *g;
12373         uint32_t i;
12374
12375         /* Pipeline table instructions. */
12376         for (i = 0; i < p->n_instructions; i++) {
12377                 struct instruction *instr = &p->instructions[i];
12378
12379                 if (instr->type == INSTR_TABLE)
12380                         instr->type = INSTR_TABLE_AF;
12381
12382                 if (instr->type == INSTR_LEARNER)
12383                         instr->type = INSTR_LEARNER_AF;
12384         }
12385
12386         /* Pipeline custom instructions. */
12387         i = 0;
12388         TAILQ_FOREACH(g, igl, node) {
12389                 struct instruction *instr = &p->instructions[g->first_instr_id];
12390                 uint32_t j;
12391
12392                 if (g->first_instr_id == g->last_instr_id)
12393                         continue;
12394
12395                 /* Install a new custom instruction. */
12396                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12397
12398                 /* First instruction of the group: change its type to the new custom instruction. */
12399                 instr->type = INSTR_CUSTOM_0 + i;
12400
12401                 /* All the subsequent instructions of the group: invalidate. */
12402                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12403                         struct instruction_data *data = &p->instruction_data[j];
12404
12405                         data->invalid = 1;
12406                 }
12407
12408                 i++;
12409         }
12410
12411         /* Remove the invalidated instructions. */
12412         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12413
12414         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12415          * instructions that are the only instruction within their group, so they were left
12416          * unmodified).
12417          */
12418         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12419 }
12420
12421 static int
12422 pipeline_compile(struct rte_swx_pipeline *p)
12423 {
12424         struct instruction_group_list *igl = NULL;
12425         int status = 0;
12426
12427         igl = instruction_group_list_create(p);
12428         if (!igl) {
12429                 status = -ENOMEM;
12430                 goto free;
12431         }
12432
12433         /* Code generation. */
12434         status = pipeline_codegen(p, igl);
12435         if (status)
12436                 goto free;
12437
12438         /* Build and load the shared object library. */
12439         status = pipeline_libload(p, igl);
12440         if (status)
12441                 goto free;
12442
12443         /* Adjust instructions. */
12444         status = pipeline_adjust_check(p, igl);
12445         if (status)
12446                 goto free;
12447
12448         pipeline_adjust(p, igl);
12449
12450 free:
12451         instruction_group_list_free(igl);
12452
12453         return status;
12454 }