lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8 #include <dlfcn.h>
   9
  10 #include "rte_swx_pipeline_internal.h"
  11
  12 #define CHECK(condition, err_code)                                             \
  13 do {                                                                           \
  14         if (!(condition))                                                      \
  15                 return -(err_code);                                            \
  16 } while (0)
  17
  18 #define CHECK_NAME(name, err_code)                                             \
  19         CHECK((name) &&                                                        \
  20               (name)[0] &&                                                     \
  21               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  22               err_code)
  23
  24 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  25         CHECK((instr) &&                                                       \
  26               (instr)[0] &&                                                    \
  27               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  28                RTE_SWX_INSTRUCTION_SIZE),                                      \
  29               err_code)
  30
  31 /*
  32  * Environment.
  33  */
  34 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  35
  36 #include <rte_malloc.h>
  37
  38 static void *
  39 env_malloc(size_t size, size_t alignment, int numa_node)
  40 {
  41         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  42 }
  43
  44 static void
  45 env_free(void *start, size_t size __rte_unused)
  46 {
  47         rte_free(start);
  48 }
  49
  50 #else
  51
  52 #include <numa.h>
  53
  54 static void *
  55 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  56 {
  57         void *start;
  58
  59         if (numa_available() == -1)
  60                 return NULL;
  61
  62         start = numa_alloc_onnode(size, numa_node);
  63         if (!start)
  64                 return NULL;
  65
  66         memset(start, 0, size);
  67         return start;
  68 }
  69
  70 static void
  71 env_free(void *start, size_t size)
  72 {
  73         if (numa_available() == -1)
  74                 return;
  75
  76         numa_free(start, size);
  77 }
  78
  79 #endif
  80
  81 /*
  82  * Struct.
  83  */
  84 static struct struct_type *
  85 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  86 {
  87         struct struct_type *elem;
  88
  89         TAILQ_FOREACH(elem, &p->struct_types, node)
  90                 if (strcmp(elem->name, name) == 0)
  91                         return elem;
  92
  93         return NULL;
  94 }
  95
  96 static struct field *
  97 struct_type_field_find(struct struct_type *st, const char *name)
  98 {
  99         uint32_t i;
 100
 101         for (i = 0; i < st->n_fields; i++) {
 102                 struct field *f = &st->fields[i];
 103
 104                 if (strcmp(f->name, name) == 0)
 105                         return f;
 106         }
 107
 108         return NULL;
 109 }
 110
 111 int
 112 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 113                                       const char *name,
 114                                       struct rte_swx_field_params *fields,
 115                                       uint32_t n_fields,
 116                                       int last_field_has_variable_size)
 117 {
 118         struct struct_type *st;
 119         uint32_t i;
 120
 121         CHECK(p, EINVAL);
 122         CHECK_NAME(name, EINVAL);
 123         CHECK(fields, EINVAL);
 124         CHECK(n_fields, EINVAL);
 125
 126         for (i = 0; i < n_fields; i++) {
 127                 struct rte_swx_field_params *f = &fields[i];
 128                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 129                 uint32_t j;
 130
 131                 CHECK_NAME(f->name, EINVAL);
 132                 CHECK(f->n_bits, EINVAL);
 133                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 134                 CHECK((f->n_bits & 7) == 0, EINVAL);
 135
 136                 for (j = 0; j < i; j++) {
 137                         struct rte_swx_field_params *f_prev = &fields[j];
 138
 139                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 140                 }
 141         }
 142
 143         CHECK(!struct_type_find(p, name), EEXIST);
 144
 145         /* Node allocation. */
 146         st = calloc(1, sizeof(struct struct_type));
 147         CHECK(st, ENOMEM);
 148
 149         st->fields = calloc(n_fields, sizeof(struct field));
 150         if (!st->fields) {
 151                 free(st);
 152                 CHECK(0, ENOMEM);
 153         }
 154
 155         /* Node initialization. */
 156         strcpy(st->name, name);
 157         for (i = 0; i < n_fields; i++) {
 158                 struct field *dst = &st->fields[i];
 159                 struct rte_swx_field_params *src = &fields[i];
 160                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 161
 162                 strcpy(dst->name, src->name);
 163                 dst->n_bits = src->n_bits;
 164                 dst->offset = st->n_bits;
 165                 dst->var_size = var_size;
 166
 167                 st->n_bits += src->n_bits;
 168                 st->n_bits_min += var_size ? 0 : src->n_bits;
 169         }
 170         st->n_fields = n_fields;
 171         st->var_size = last_field_has_variable_size;
 172
 173         /* Node add to tailq. */
 174         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 175
 176         return 0;
 177 }
 178
 179 static int
 180 struct_build(struct rte_swx_pipeline *p)
 181 {
 182         uint32_t i;
 183
 184         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 185                 struct thread *t = &p->threads[i];
 186
 187                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 188                 CHECK(t->structs, ENOMEM);
 189         }
 190
 191         return 0;
 192 }
 193
 194 static void
 195 struct_build_free(struct rte_swx_pipeline *p)
 196 {
 197         uint32_t i;
 198
 199         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 200                 struct thread *t = &p->threads[i];
 201
 202                 free(t->structs);
 203                 t->structs = NULL;
 204         }
 205 }
 206
 207 static void
 208 struct_free(struct rte_swx_pipeline *p)
 209 {
 210         struct_build_free(p);
 211
 212         /* Struct types. */
 213         for ( ; ; ) {
 214                 struct struct_type *elem;
 215
 216                 elem = TAILQ_FIRST(&p->struct_types);
 217                 if (!elem)
 218                         break;
 219
 220                 TAILQ_REMOVE(&p->struct_types, elem, node);
 221                 free(elem->fields);
 222                 free(elem);
 223         }
 224 }
 225
 226 /*
 227  * Input port.
 228  */
 229 static struct port_in_type *
 230 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 231 {
 232         struct port_in_type *elem;
 233
 234         if (!name)
 235                 return NULL;
 236
 237         TAILQ_FOREACH(elem, &p->port_in_types, node)
 238                 if (strcmp(elem->name, name) == 0)
 239                         return elem;
 240
 241         return NULL;
 242 }
 243
 244 int
 245 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 246                                        const char *name,
 247                                        struct rte_swx_port_in_ops *ops)
 248 {
 249         struct port_in_type *elem;
 250
 251         CHECK(p, EINVAL);
 252         CHECK_NAME(name, EINVAL);
 253         CHECK(ops, EINVAL);
 254         CHECK(ops->create, EINVAL);
 255         CHECK(ops->free, EINVAL);
 256         CHECK(ops->pkt_rx, EINVAL);
 257         CHECK(ops->stats_read, EINVAL);
 258
 259         CHECK(!port_in_type_find(p, name), EEXIST);
 260
 261         /* Node allocation. */
 262         elem = calloc(1, sizeof(struct port_in_type));
 263         CHECK(elem, ENOMEM);
 264
 265         /* Node initialization. */
 266         strcpy(elem->name, name);
 267         memcpy(&elem->ops, ops, sizeof(*ops));
 268
 269         /* Node add to tailq. */
 270         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 271
 272         return 0;
 273 }
 274
 275 static struct port_in *
 276 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 277 {
 278         struct port_in *port;
 279
 280         TAILQ_FOREACH(port, &p->ports_in, node)
 281                 if (port->id == port_id)
 282                         return port;
 283
 284         return NULL;
 285 }
 286
 287 int
 288 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 289                                 uint32_t port_id,
 290                                 const char *port_type_name,
 291                                 void *args)
 292 {
 293         struct port_in_type *type = NULL;
 294         struct port_in *port = NULL;
 295         void *obj = NULL;
 296
 297         CHECK(p, EINVAL);
 298
 299         CHECK(!port_in_find(p, port_id), EINVAL);
 300
 301         CHECK_NAME(port_type_name, EINVAL);
 302         type = port_in_type_find(p, port_type_name);
 303         CHECK(type, EINVAL);
 304
 305         obj = type->ops.create(args);
 306         CHECK(obj, ENODEV);
 307
 308         /* Node allocation. */
 309         port = calloc(1, sizeof(struct port_in));
 310         CHECK(port, ENOMEM);
 311
 312         /* Node initialization. */
 313         port->type = type;
 314         port->obj = obj;
 315         port->id = port_id;
 316
 317         /* Node add to tailq. */
 318         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 319         if (p->n_ports_in < port_id + 1)
 320                 p->n_ports_in = port_id + 1;
 321
 322         return 0;
 323 }
 324
 325 static int
 326 port_in_build(struct rte_swx_pipeline *p)
 327 {
 328         struct port_in *port;
 329         uint32_t i;
 330
 331         CHECK(p->n_ports_in, EINVAL);
 332         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 333
 334         for (i = 0; i < p->n_ports_in; i++)
 335                 CHECK(port_in_find(p, i), EINVAL);
 336
 337         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 338         CHECK(p->in, ENOMEM);
 339
 340         TAILQ_FOREACH(port, &p->ports_in, node) {
 341                 struct port_in_runtime *in = &p->in[port->id];
 342
 343                 in->pkt_rx = port->type->ops.pkt_rx;
 344                 in->obj = port->obj;
 345         }
 346
 347         return 0;
 348 }
 349
 350 static void
 351 port_in_build_free(struct rte_swx_pipeline *p)
 352 {
 353         free(p->in);
 354         p->in = NULL;
 355 }
 356
 357 static void
 358 port_in_free(struct rte_swx_pipeline *p)
 359 {
 360         port_in_build_free(p);
 361
 362         /* Input ports. */
 363         for ( ; ; ) {
 364                 struct port_in *port;
 365
 366                 port = TAILQ_FIRST(&p->ports_in);
 367                 if (!port)
 368                         break;
 369
 370                 TAILQ_REMOVE(&p->ports_in, port, node);
 371                 port->type->ops.free(port->obj);
 372                 free(port);
 373         }
 374
 375         /* Input port types. */
 376         for ( ; ; ) {
 377                 struct port_in_type *elem;
 378
 379                 elem = TAILQ_FIRST(&p->port_in_types);
 380                 if (!elem)
 381                         break;
 382
 383                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 384                 free(elem);
 385         }
 386 }
 387
 388 /*
 389  * Output port.
 390  */
 391 static struct port_out_type *
 392 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 393 {
 394         struct port_out_type *elem;
 395
 396         if (!name)
 397                 return NULL;
 398
 399         TAILQ_FOREACH(elem, &p->port_out_types, node)
 400                 if (!strcmp(elem->name, name))
 401                         return elem;
 402
 403         return NULL;
 404 }
 405
 406 int
 407 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 408                                         const char *name,
 409                                         struct rte_swx_port_out_ops *ops)
 410 {
 411         struct port_out_type *elem;
 412
 413         CHECK(p, EINVAL);
 414         CHECK_NAME(name, EINVAL);
 415         CHECK(ops, EINVAL);
 416         CHECK(ops->create, EINVAL);
 417         CHECK(ops->free, EINVAL);
 418         CHECK(ops->pkt_tx, EINVAL);
 419         CHECK(ops->stats_read, EINVAL);
 420
 421         CHECK(!port_out_type_find(p, name), EEXIST);
 422
 423         /* Node allocation. */
 424         elem = calloc(1, sizeof(struct port_out_type));
 425         CHECK(elem, ENOMEM);
 426
 427         /* Node initialization. */
 428         strcpy(elem->name, name);
 429         memcpy(&elem->ops, ops, sizeof(*ops));
 430
 431         /* Node add to tailq. */
 432         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 433
 434         return 0;
 435 }
 436
 437 static struct port_out *
 438 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 439 {
 440         struct port_out *port;
 441
 442         TAILQ_FOREACH(port, &p->ports_out, node)
 443                 if (port->id == port_id)
 444                         return port;
 445
 446         return NULL;
 447 }
 448
 449 int
 450 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 451                                  uint32_t port_id,
 452                                  const char *port_type_name,
 453                                  void *args)
 454 {
 455         struct port_out_type *type = NULL;
 456         struct port_out *port = NULL;
 457         void *obj = NULL;
 458
 459         CHECK(p, EINVAL);
 460
 461         CHECK(!port_out_find(p, port_id), EINVAL);
 462
 463         CHECK_NAME(port_type_name, EINVAL);
 464         type = port_out_type_find(p, port_type_name);
 465         CHECK(type, EINVAL);
 466
 467         obj = type->ops.create(args);
 468         CHECK(obj, ENODEV);
 469
 470         /* Node allocation. */
 471         port = calloc(1, sizeof(struct port_out));
 472         CHECK(port, ENOMEM);
 473
 474         /* Node initialization. */
 475         port->type = type;
 476         port->obj = obj;
 477         port->id = port_id;
 478
 479         /* Node add to tailq. */
 480         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 481         if (p->n_ports_out < port_id + 1)
 482                 p->n_ports_out = port_id + 1;
 483
 484         return 0;
 485 }
 486
 487 static int
 488 port_out_build(struct rte_swx_pipeline *p)
 489 {
 490         struct port_out *port;
 491         uint32_t i;
 492
 493         CHECK(p->n_ports_out, EINVAL);
 494
 495         for (i = 0; i < p->n_ports_out; i++)
 496                 CHECK(port_out_find(p, i), EINVAL);
 497
 498         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 499         CHECK(p->out, ENOMEM);
 500
 501         TAILQ_FOREACH(port, &p->ports_out, node) {
 502                 struct port_out_runtime *out = &p->out[port->id];
 503
 504                 out->pkt_tx = port->type->ops.pkt_tx;
 505                 out->flush = port->type->ops.flush;
 506                 out->obj = port->obj;
 507         }
 508
 509         return 0;
 510 }
 511
 512 static void
 513 port_out_build_free(struct rte_swx_pipeline *p)
 514 {
 515         free(p->out);
 516         p->out = NULL;
 517 }
 518
 519 static void
 520 port_out_free(struct rte_swx_pipeline *p)
 521 {
 522         port_out_build_free(p);
 523
 524         /* Output ports. */
 525         for ( ; ; ) {
 526                 struct port_out *port;
 527
 528                 port = TAILQ_FIRST(&p->ports_out);
 529                 if (!port)
 530                         break;
 531
 532                 TAILQ_REMOVE(&p->ports_out, port, node);
 533                 port->type->ops.free(port->obj);
 534                 free(port);
 535         }
 536
 537         /* Output port types. */
 538         for ( ; ; ) {
 539                 struct port_out_type *elem;
 540
 541                 elem = TAILQ_FIRST(&p->port_out_types);
 542                 if (!elem)
 543                         break;
 544
 545                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 546                 free(elem);
 547         }
 548 }
 549
 550 /*
 551  * Extern object.
 552  */
 553 static struct extern_type *
 554 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 555 {
 556         struct extern_type *elem;
 557
 558         TAILQ_FOREACH(elem, &p->extern_types, node)
 559                 if (strcmp(elem->name, name) == 0)
 560                         return elem;
 561
 562         return NULL;
 563 }
 564
 565 static struct extern_type_member_func *
 566 extern_type_member_func_find(struct extern_type *type, const char *name)
 567 {
 568         struct extern_type_member_func *elem;
 569
 570         TAILQ_FOREACH(elem, &type->funcs, node)
 571                 if (strcmp(elem->name, name) == 0)
 572                         return elem;
 573
 574         return NULL;
 575 }
 576
 577 static struct extern_obj *
 578 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 579 {
 580         struct extern_obj *elem;
 581
 582         TAILQ_FOREACH(elem, &p->extern_objs, node)
 583                 if (strcmp(elem->name, name) == 0)
 584                         return elem;
 585
 586         return NULL;
 587 }
 588
 589 static struct extern_type_member_func *
 590 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 591                              const char *name,
 592                              struct extern_obj **obj)
 593 {
 594         struct extern_obj *object;
 595         struct extern_type_member_func *func;
 596         char *object_name, *func_name;
 597
 598         if (name[0] != 'e' || name[1] != '.')
 599                 return NULL;
 600
 601         object_name = strdup(&name[2]);
 602         if (!object_name)
 603                 return NULL;
 604
 605         func_name = strchr(object_name, '.');
 606         if (!func_name) {
 607                 free(object_name);
 608                 return NULL;
 609         }
 610
 611         *func_name = 0;
 612         func_name++;
 613
 614         object = extern_obj_find(p, object_name);
 615         if (!object) {
 616                 free(object_name);
 617                 return NULL;
 618         }
 619
 620         func = extern_type_member_func_find(object->type, func_name);
 621         if (!func) {
 622                 free(object_name);
 623                 return NULL;
 624         }
 625
 626         if (obj)
 627                 *obj = object;
 628
 629         free(object_name);
 630         return func;
 631 }
 632
 633 static struct field *
 634 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 635                                const char *name,
 636                                struct extern_obj **object)
 637 {
 638         struct extern_obj *obj;
 639         struct field *f;
 640         char *obj_name, *field_name;
 641
 642         if ((name[0] != 'e') || (name[1] != '.'))
 643                 return NULL;
 644
 645         obj_name = strdup(&name[2]);
 646         if (!obj_name)
 647                 return NULL;
 648
 649         field_name = strchr(obj_name, '.');
 650         if (!field_name) {
 651                 free(obj_name);
 652                 return NULL;
 653         }
 654
 655         *field_name = 0;
 656         field_name++;
 657
 658         obj = extern_obj_find(p, obj_name);
 659         if (!obj) {
 660                 free(obj_name);
 661                 return NULL;
 662         }
 663
 664         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 665         if (!f) {
 666                 free(obj_name);
 667                 return NULL;
 668         }
 669
 670         if (object)
 671                 *object = obj;
 672
 673         free(obj_name);
 674         return f;
 675 }
 676
 677 int
 678 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 679         const char *name,
 680         const char *mailbox_struct_type_name,
 681         rte_swx_extern_type_constructor_t constructor,
 682         rte_swx_extern_type_destructor_t destructor)
 683 {
 684         struct extern_type *elem;
 685         struct struct_type *mailbox_struct_type;
 686
 687         CHECK(p, EINVAL);
 688
 689         CHECK_NAME(name, EINVAL);
 690         CHECK(!extern_type_find(p, name), EEXIST);
 691
 692         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 693         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 694         CHECK(mailbox_struct_type, EINVAL);
 695         CHECK(!mailbox_struct_type->var_size, EINVAL);
 696
 697         CHECK(constructor, EINVAL);
 698         CHECK(destructor, EINVAL);
 699
 700         /* Node allocation. */
 701         elem = calloc(1, sizeof(struct extern_type));
 702         CHECK(elem, ENOMEM);
 703
 704         /* Node initialization. */
 705         strcpy(elem->name, name);
 706         elem->mailbox_struct_type = mailbox_struct_type;
 707         elem->constructor = constructor;
 708         elem->destructor = destructor;
 709         TAILQ_INIT(&elem->funcs);
 710
 711         /* Node add to tailq. */
 712         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 713
 714         return 0;
 715 }
 716
 717 int
 718 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 719         const char *extern_type_name,
 720         const char *name,
 721         rte_swx_extern_type_member_func_t member_func)
 722 {
 723         struct extern_type *type;
 724         struct extern_type_member_func *type_member;
 725
 726         CHECK(p, EINVAL);
 727
 728         CHECK_NAME(extern_type_name, EINVAL);
 729         type = extern_type_find(p, extern_type_name);
 730         CHECK(type, EINVAL);
 731         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 732
 733         CHECK_NAME(name, EINVAL);
 734         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 735
 736         CHECK(member_func, EINVAL);
 737
 738         /* Node allocation. */
 739         type_member = calloc(1, sizeof(struct extern_type_member_func));
 740         CHECK(type_member, ENOMEM);
 741
 742         /* Node initialization. */
 743         strcpy(type_member->name, name);
 744         type_member->func = member_func;
 745         type_member->id = type->n_funcs;
 746
 747         /* Node add to tailq. */
 748         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 749         type->n_funcs++;
 750
 751         return 0;
 752 }
 753
 754 int
 755 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 756                                       const char *extern_type_name,
 757                                       const char *name,
 758                                       const char *args)
 759 {
 760         struct extern_type *type;
 761         struct extern_obj *obj;
 762         void *obj_handle;
 763
 764         CHECK(p, EINVAL);
 765
 766         CHECK_NAME(extern_type_name, EINVAL);
 767         type = extern_type_find(p, extern_type_name);
 768         CHECK(type, EINVAL);
 769
 770         CHECK_NAME(name, EINVAL);
 771         CHECK(!extern_obj_find(p, name), EEXIST);
 772
 773         /* Node allocation. */
 774         obj = calloc(1, sizeof(struct extern_obj));
 775         CHECK(obj, ENOMEM);
 776
 777         /* Object construction. */
 778         obj_handle = type->constructor(args);
 779         if (!obj_handle) {
 780                 free(obj);
 781                 CHECK(0, ENODEV);
 782         }
 783
 784         /* Node initialization. */
 785         strcpy(obj->name, name);
 786         obj->type = type;
 787         obj->obj = obj_handle;
 788         obj->struct_id = p->n_structs;
 789         obj->id = p->n_extern_objs;
 790
 791         /* Node add to tailq. */
 792         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 793         p->n_extern_objs++;
 794         p->n_structs++;
 795
 796         return 0;
 797 }
 798
 799 static int
 800 extern_obj_build(struct rte_swx_pipeline *p)
 801 {
 802         uint32_t i;
 803
 804         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 805                 struct thread *t = &p->threads[i];
 806                 struct extern_obj *obj;
 807
 808                 t->extern_objs = calloc(p->n_extern_objs,
 809                                         sizeof(struct extern_obj_runtime));
 810                 CHECK(t->extern_objs, ENOMEM);
 811
 812                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 813                         struct extern_obj_runtime *r =
 814                                 &t->extern_objs[obj->id];
 815                         struct extern_type_member_func *func;
 816                         uint32_t mailbox_size =
 817                                 obj->type->mailbox_struct_type->n_bits / 8;
 818
 819                         r->obj = obj->obj;
 820
 821                         r->mailbox = calloc(1, mailbox_size);
 822                         CHECK(r->mailbox, ENOMEM);
 823
 824                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 825                                 r->funcs[func->id] = func->func;
 826
 827                         t->structs[obj->struct_id] = r->mailbox;
 828                 }
 829         }
 830
 831         return 0;
 832 }
 833
 834 static void
 835 extern_obj_build_free(struct rte_swx_pipeline *p)
 836 {
 837         uint32_t i;
 838
 839         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 840                 struct thread *t = &p->threads[i];
 841                 uint32_t j;
 842
 843                 if (!t->extern_objs)
 844                         continue;
 845
 846                 for (j = 0; j < p->n_extern_objs; j++) {
 847                         struct extern_obj_runtime *r = &t->extern_objs[j];
 848
 849                         free(r->mailbox);
 850                 }
 851
 852                 free(t->extern_objs);
 853                 t->extern_objs = NULL;
 854         }
 855 }
 856
 857 static void
 858 extern_obj_free(struct rte_swx_pipeline *p)
 859 {
 860         extern_obj_build_free(p);
 861
 862         /* Extern objects. */
 863         for ( ; ; ) {
 864                 struct extern_obj *elem;
 865
 866                 elem = TAILQ_FIRST(&p->extern_objs);
 867                 if (!elem)
 868                         break;
 869
 870                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 871                 if (elem->obj)
 872                         elem->type->destructor(elem->obj);
 873                 free(elem);
 874         }
 875
 876         /* Extern types. */
 877         for ( ; ; ) {
 878                 struct extern_type *elem;
 879
 880                 elem = TAILQ_FIRST(&p->extern_types);
 881                 if (!elem)
 882                         break;
 883
 884                 TAILQ_REMOVE(&p->extern_types, elem, node);
 885
 886                 for ( ; ; ) {
 887                         struct extern_type_member_func *func;
 888
 889                         func = TAILQ_FIRST(&elem->funcs);
 890                         if (!func)
 891                                 break;
 892
 893                         TAILQ_REMOVE(&elem->funcs, func, node);
 894                         free(func);
 895                 }
 896
 897                 free(elem);
 898         }
 899 }
 900
 901 /*
 902  * Extern function.
 903  */
 904 static struct extern_func *
 905 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 906 {
 907         struct extern_func *elem;
 908
 909         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 910                 if (strcmp(elem->name, name) == 0)
 911                         return elem;
 912
 913         return NULL;
 914 }
 915
 916 static struct extern_func *
 917 extern_func_parse(struct rte_swx_pipeline *p,
 918                   const char *name)
 919 {
 920         if (name[0] != 'f' || name[1] != '.')
 921                 return NULL;
 922
 923         return extern_func_find(p, &name[2]);
 924 }
 925
 926 static struct field *
 927 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 928                                 const char *name,
 929                                 struct extern_func **function)
 930 {
 931         struct extern_func *func;
 932         struct field *f;
 933         char *func_name, *field_name;
 934
 935         if ((name[0] != 'f') || (name[1] != '.'))
 936                 return NULL;
 937
 938         func_name = strdup(&name[2]);
 939         if (!func_name)
 940                 return NULL;
 941
 942         field_name = strchr(func_name, '.');
 943         if (!field_name) {
 944                 free(func_name);
 945                 return NULL;
 946         }
 947
 948         *field_name = 0;
 949         field_name++;
 950
 951         func = extern_func_find(p, func_name);
 952         if (!func) {
 953                 free(func_name);
 954                 return NULL;
 955         }
 956
 957         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 958         if (!f) {
 959                 free(func_name);
 960                 return NULL;
 961         }
 962
 963         if (function)
 964                 *function = func;
 965
 966         free(func_name);
 967         return f;
 968 }
 969
 970 int
 971 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 972                                       const char *name,
 973                                       const char *mailbox_struct_type_name,
 974                                       rte_swx_extern_func_t func)
 975 {
 976         struct extern_func *f;
 977         struct struct_type *mailbox_struct_type;
 978
 979         CHECK(p, EINVAL);
 980
 981         CHECK_NAME(name, EINVAL);
 982         CHECK(!extern_func_find(p, name), EEXIST);
 983
 984         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 985         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 986         CHECK(mailbox_struct_type, EINVAL);
 987         CHECK(!mailbox_struct_type->var_size, EINVAL);
 988
 989         CHECK(func, EINVAL);
 990
 991         /* Node allocation. */
 992         f = calloc(1, sizeof(struct extern_func));
 993         CHECK(func, ENOMEM);
 994
 995         /* Node initialization. */
 996         strcpy(f->name, name);
 997         f->mailbox_struct_type = mailbox_struct_type;
 998         f->func = func;
 999         f->struct_id = p->n_structs;
1000         f->id = p->n_extern_funcs;
1001
1002         /* Node add to tailq. */
1003         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1004         p->n_extern_funcs++;
1005         p->n_structs++;
1006
1007         return 0;
1008 }
1009
1010 static int
1011 extern_func_build(struct rte_swx_pipeline *p)
1012 {
1013         uint32_t i;
1014
1015         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1016                 struct thread *t = &p->threads[i];
1017                 struct extern_func *func;
1018
1019                 /* Memory allocation. */
1020                 t->extern_funcs = calloc(p->n_extern_funcs,
1021                                          sizeof(struct extern_func_runtime));
1022                 CHECK(t->extern_funcs, ENOMEM);
1023
1024                 /* Extern function. */
1025                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1026                         struct extern_func_runtime *r =
1027                                 &t->extern_funcs[func->id];
1028                         uint32_t mailbox_size =
1029                                 func->mailbox_struct_type->n_bits / 8;
1030
1031                         r->func = func->func;
1032
1033                         r->mailbox = calloc(1, mailbox_size);
1034                         CHECK(r->mailbox, ENOMEM);
1035
1036                         t->structs[func->struct_id] = r->mailbox;
1037                 }
1038         }
1039
1040         return 0;
1041 }
1042
1043 static void
1044 extern_func_build_free(struct rte_swx_pipeline *p)
1045 {
1046         uint32_t i;
1047
1048         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1049                 struct thread *t = &p->threads[i];
1050                 uint32_t j;
1051
1052                 if (!t->extern_funcs)
1053                         continue;
1054
1055                 for (j = 0; j < p->n_extern_funcs; j++) {
1056                         struct extern_func_runtime *r = &t->extern_funcs[j];
1057
1058                         free(r->mailbox);
1059                 }
1060
1061                 free(t->extern_funcs);
1062                 t->extern_funcs = NULL;
1063         }
1064 }
1065
1066 static void
1067 extern_func_free(struct rte_swx_pipeline *p)
1068 {
1069         extern_func_build_free(p);
1070
1071         for ( ; ; ) {
1072                 struct extern_func *elem;
1073
1074                 elem = TAILQ_FIRST(&p->extern_funcs);
1075                 if (!elem)
1076                         break;
1077
1078                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1079                 free(elem);
1080         }
1081 }
1082
1083 /*
1084  * Header.
1085  */
1086 static struct header *
1087 header_find(struct rte_swx_pipeline *p, const char *name)
1088 {
1089         struct header *elem;
1090
1091         TAILQ_FOREACH(elem, &p->headers, node)
1092                 if (strcmp(elem->name, name) == 0)
1093                         return elem;
1094
1095         return NULL;
1096 }
1097
1098 static struct header *
1099 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1100 {
1101         struct header *elem;
1102
1103         TAILQ_FOREACH(elem, &p->headers, node)
1104                 if (elem->struct_id == struct_id)
1105                         return elem;
1106
1107         return NULL;
1108 }
1109
1110 static struct header *
1111 header_parse(struct rte_swx_pipeline *p,
1112              const char *name)
1113 {
1114         if (name[0] != 'h' || name[1] != '.')
1115                 return NULL;
1116
1117         return header_find(p, &name[2]);
1118 }
1119
1120 static struct field *
1121 header_field_parse(struct rte_swx_pipeline *p,
1122                    const char *name,
1123                    struct header **header)
1124 {
1125         struct header *h;
1126         struct field *f;
1127         char *header_name, *field_name;
1128
1129         if ((name[0] != 'h') || (name[1] != '.'))
1130                 return NULL;
1131
1132         header_name = strdup(&name[2]);
1133         if (!header_name)
1134                 return NULL;
1135
1136         field_name = strchr(header_name, '.');
1137         if (!field_name) {
1138                 free(header_name);
1139                 return NULL;
1140         }
1141
1142         *field_name = 0;
1143         field_name++;
1144
1145         h = header_find(p, header_name);
1146         if (!h) {
1147                 free(header_name);
1148                 return NULL;
1149         }
1150
1151         f = struct_type_field_find(h->st, field_name);
1152         if (!f) {
1153                 free(header_name);
1154                 return NULL;
1155         }
1156
1157         if (header)
1158                 *header = h;
1159
1160         free(header_name);
1161         return f;
1162 }
1163
1164 int
1165 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1166                                         const char *name,
1167                                         const char *struct_type_name)
1168 {
1169         struct struct_type *st;
1170         struct header *h;
1171         size_t n_headers_max;
1172
1173         CHECK(p, EINVAL);
1174         CHECK_NAME(name, EINVAL);
1175         CHECK_NAME(struct_type_name, EINVAL);
1176
1177         CHECK(!header_find(p, name), EEXIST);
1178
1179         st = struct_type_find(p, struct_type_name);
1180         CHECK(st, EINVAL);
1181
1182         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1183         CHECK(p->n_headers < n_headers_max, ENOSPC);
1184
1185         /* Node allocation. */
1186         h = calloc(1, sizeof(struct header));
1187         CHECK(h, ENOMEM);
1188
1189         /* Node initialization. */
1190         strcpy(h->name, name);
1191         h->st = st;
1192         h->struct_id = p->n_structs;
1193         h->id = p->n_headers;
1194
1195         /* Node add to tailq. */
1196         TAILQ_INSERT_TAIL(&p->headers, h, node);
1197         p->n_headers++;
1198         p->n_structs++;
1199
1200         return 0;
1201 }
1202
1203 static int
1204 header_build(struct rte_swx_pipeline *p)
1205 {
1206         struct header *h;
1207         uint32_t n_bytes = 0, i;
1208
1209         TAILQ_FOREACH(h, &p->headers, node) {
1210                 n_bytes += h->st->n_bits / 8;
1211         }
1212
1213         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1214                 struct thread *t = &p->threads[i];
1215                 uint32_t offset = 0;
1216
1217                 t->headers = calloc(p->n_headers,
1218                                     sizeof(struct header_runtime));
1219                 CHECK(t->headers, ENOMEM);
1220
1221                 t->headers_out = calloc(p->n_headers,
1222                                         sizeof(struct header_out_runtime));
1223                 CHECK(t->headers_out, ENOMEM);
1224
1225                 t->header_storage = calloc(1, n_bytes);
1226                 CHECK(t->header_storage, ENOMEM);
1227
1228                 t->header_out_storage = calloc(1, n_bytes);
1229                 CHECK(t->header_out_storage, ENOMEM);
1230
1231                 TAILQ_FOREACH(h, &p->headers, node) {
1232                         uint8_t *header_storage;
1233                         uint32_t n_bytes =  h->st->n_bits / 8;
1234
1235                         header_storage = &t->header_storage[offset];
1236                         offset += n_bytes;
1237
1238                         t->headers[h->id].ptr0 = header_storage;
1239                         t->headers[h->id].n_bytes = n_bytes;
1240
1241                         t->structs[h->struct_id] = header_storage;
1242                 }
1243         }
1244
1245         return 0;
1246 }
1247
1248 static void
1249 header_build_free(struct rte_swx_pipeline *p)
1250 {
1251         uint32_t i;
1252
1253         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1254                 struct thread *t = &p->threads[i];
1255
1256                 free(t->headers_out);
1257                 t->headers_out = NULL;
1258
1259                 free(t->headers);
1260                 t->headers = NULL;
1261
1262                 free(t->header_out_storage);
1263                 t->header_out_storage = NULL;
1264
1265                 free(t->header_storage);
1266                 t->header_storage = NULL;
1267         }
1268 }
1269
1270 static void
1271 header_free(struct rte_swx_pipeline *p)
1272 {
1273         header_build_free(p);
1274
1275         for ( ; ; ) {
1276                 struct header *elem;
1277
1278                 elem = TAILQ_FIRST(&p->headers);
1279                 if (!elem)
1280                         break;
1281
1282                 TAILQ_REMOVE(&p->headers, elem, node);
1283                 free(elem);
1284         }
1285 }
1286
1287 /*
1288  * Meta-data.
1289  */
1290 static struct field *
1291 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1292 {
1293         if (!p->metadata_st)
1294                 return NULL;
1295
1296         if (name[0] != 'm' || name[1] != '.')
1297                 return NULL;
1298
1299         return struct_type_field_find(p->metadata_st, &name[2]);
1300 }
1301
1302 int
1303 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1304                                           const char *struct_type_name)
1305 {
1306         struct struct_type *st = NULL;
1307
1308         CHECK(p, EINVAL);
1309
1310         CHECK_NAME(struct_type_name, EINVAL);
1311         st  = struct_type_find(p, struct_type_name);
1312         CHECK(st, EINVAL);
1313         CHECK(!st->var_size, EINVAL);
1314         CHECK(!p->metadata_st, EINVAL);
1315
1316         p->metadata_st = st;
1317         p->metadata_struct_id = p->n_structs;
1318
1319         p->n_structs++;
1320
1321         return 0;
1322 }
1323
1324 static int
1325 metadata_build(struct rte_swx_pipeline *p)
1326 {
1327         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1328         uint32_t i;
1329
1330         /* Thread-level initialization. */
1331         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1332                 struct thread *t = &p->threads[i];
1333                 uint8_t *metadata;
1334
1335                 metadata = calloc(1, n_bytes);
1336                 CHECK(metadata, ENOMEM);
1337
1338                 t->metadata = metadata;
1339                 t->structs[p->metadata_struct_id] = metadata;
1340         }
1341
1342         return 0;
1343 }
1344
1345 static void
1346 metadata_build_free(struct rte_swx_pipeline *p)
1347 {
1348         uint32_t i;
1349
1350         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1351                 struct thread *t = &p->threads[i];
1352
1353                 free(t->metadata);
1354                 t->metadata = NULL;
1355         }
1356 }
1357
1358 static void
1359 metadata_free(struct rte_swx_pipeline *p)
1360 {
1361         metadata_build_free(p);
1362 }
1363
1364 /*
1365  * Instruction.
1366  */
1367 static int
1368 instruction_is_tx(enum instruction_type type)
1369 {
1370         switch (type) {
1371         case INSTR_TX:
1372         case INSTR_TX_I:
1373                 return 1;
1374
1375         default:
1376                 return 0;
1377         }
1378 }
1379
1380 static int
1381 instruction_does_tx(struct instruction *instr)
1382 {
1383         switch (instr->type) {
1384         case INSTR_TX:
1385         case INSTR_TX_I:
1386         case INSTR_HDR_EMIT_TX:
1387         case INSTR_HDR_EMIT2_TX:
1388         case INSTR_HDR_EMIT3_TX:
1389         case INSTR_HDR_EMIT4_TX:
1390         case INSTR_HDR_EMIT5_TX:
1391         case INSTR_HDR_EMIT6_TX:
1392         case INSTR_HDR_EMIT7_TX:
1393         case INSTR_HDR_EMIT8_TX:
1394                 return 1;
1395         default:
1396                 return 0;
1397         }
1398 }
1399
1400 static int
1401 instruction_is_jmp(struct instruction *instr)
1402 {
1403         switch (instr->type) {
1404         case INSTR_JMP:
1405         case INSTR_JMP_VALID:
1406         case INSTR_JMP_INVALID:
1407         case INSTR_JMP_HIT:
1408         case INSTR_JMP_MISS:
1409         case INSTR_JMP_ACTION_HIT:
1410         case INSTR_JMP_ACTION_MISS:
1411         case INSTR_JMP_EQ:
1412         case INSTR_JMP_EQ_MH:
1413         case INSTR_JMP_EQ_HM:
1414         case INSTR_JMP_EQ_HH:
1415         case INSTR_JMP_EQ_I:
1416         case INSTR_JMP_NEQ:
1417         case INSTR_JMP_NEQ_MH:
1418         case INSTR_JMP_NEQ_HM:
1419         case INSTR_JMP_NEQ_HH:
1420         case INSTR_JMP_NEQ_I:
1421         case INSTR_JMP_LT:
1422         case INSTR_JMP_LT_MH:
1423         case INSTR_JMP_LT_HM:
1424         case INSTR_JMP_LT_HH:
1425         case INSTR_JMP_LT_MI:
1426         case INSTR_JMP_LT_HI:
1427         case INSTR_JMP_GT:
1428         case INSTR_JMP_GT_MH:
1429         case INSTR_JMP_GT_HM:
1430         case INSTR_JMP_GT_HH:
1431         case INSTR_JMP_GT_MI:
1432         case INSTR_JMP_GT_HI:
1433                 return 1;
1434
1435         default:
1436                 return 0;
1437         }
1438 }
1439
1440 static int
1441 instruction_does_thread_yield(struct instruction *instr)
1442 {
1443         switch (instr->type) {
1444         case INSTR_RX:
1445         case INSTR_TABLE:
1446         case INSTR_TABLE_AF:
1447         case INSTR_SELECTOR:
1448         case INSTR_LEARNER:
1449         case INSTR_LEARNER_AF:
1450         case INSTR_EXTERN_OBJ:
1451         case INSTR_EXTERN_FUNC:
1452                 return 1;
1453         default:
1454                 return 0;
1455         }
1456 }
1457
1458 static struct field *
1459 action_field_parse(struct action *action, const char *name);
1460
1461 static struct field *
1462 struct_field_parse(struct rte_swx_pipeline *p,
1463                    struct action *action,
1464                    const char *name,
1465                    uint32_t *struct_id)
1466 {
1467         struct field *f;
1468
1469         switch (name[0]) {
1470         case 'h':
1471         {
1472                 struct header *header;
1473
1474                 f = header_field_parse(p, name, &header);
1475                 if (!f)
1476                         return NULL;
1477
1478                 *struct_id = header->struct_id;
1479                 return f;
1480         }
1481
1482         case 'm':
1483         {
1484                 f = metadata_field_parse(p, name);
1485                 if (!f)
1486                         return NULL;
1487
1488                 *struct_id = p->metadata_struct_id;
1489                 return f;
1490         }
1491
1492         case 't':
1493         {
1494                 if (!action)
1495                         return NULL;
1496
1497                 f = action_field_parse(action, name);
1498                 if (!f)
1499                         return NULL;
1500
1501                 *struct_id = 0;
1502                 return f;
1503         }
1504
1505         case 'e':
1506         {
1507                 struct extern_obj *obj;
1508
1509                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1510                 if (!f)
1511                         return NULL;
1512
1513                 *struct_id = obj->struct_id;
1514                 return f;
1515         }
1516
1517         case 'f':
1518         {
1519                 struct extern_func *func;
1520
1521                 f = extern_func_mailbox_field_parse(p, name, &func);
1522                 if (!f)
1523                         return NULL;
1524
1525                 *struct_id = func->struct_id;
1526                 return f;
1527         }
1528
1529         default:
1530                 return NULL;
1531         }
1532 }
1533
1534 /*
1535  * rx.
1536  */
1537 static int
1538 instr_rx_translate(struct rte_swx_pipeline *p,
1539                    struct action *action,
1540                    char **tokens,
1541                    int n_tokens,
1542                    struct instruction *instr,
1543                    struct instruction_data *data __rte_unused)
1544 {
1545         struct field *f;
1546
1547         CHECK(!action, EINVAL);
1548         CHECK(n_tokens == 2, EINVAL);
1549
1550         f = metadata_field_parse(p, tokens[1]);
1551         CHECK(f, EINVAL);
1552
1553         instr->type = INSTR_RX;
1554         instr->io.io.offset = f->offset / 8;
1555         instr->io.io.n_bits = f->n_bits;
1556         return 0;
1557 }
1558
1559 /*
1560  * tx.
1561  */
1562 static int
1563 instr_tx_translate(struct rte_swx_pipeline *p,
1564                    struct action *action __rte_unused,
1565                    char **tokens,
1566                    int n_tokens,
1567                    struct instruction *instr,
1568                    struct instruction_data *data __rte_unused)
1569 {
1570         char *port = tokens[1];
1571         struct field *f;
1572         uint32_t port_val;
1573
1574         CHECK(n_tokens == 2, EINVAL);
1575
1576         f = metadata_field_parse(p, port);
1577         if (f) {
1578                 instr->type = INSTR_TX;
1579                 instr->io.io.offset = f->offset / 8;
1580                 instr->io.io.n_bits = f->n_bits;
1581                 return 0;
1582         }
1583
1584         /* TX_I. */
1585         port_val = strtoul(port, &port, 0);
1586         CHECK(!port[0], EINVAL);
1587
1588         instr->type = INSTR_TX_I;
1589         instr->io.io.val = port_val;
1590         return 0;
1591 }
1592
1593 static int
1594 instr_drop_translate(struct rte_swx_pipeline *p,
1595                      struct action *action __rte_unused,
1596                      char **tokens __rte_unused,
1597                      int n_tokens,
1598                      struct instruction *instr,
1599                      struct instruction_data *data __rte_unused)
1600 {
1601         CHECK(n_tokens == 1, EINVAL);
1602
1603         /* TX_I. */
1604         instr->type = INSTR_TX_I;
1605         instr->io.io.val = p->n_ports_out - 1;
1606         return 0;
1607 }
1608
1609 static inline void
1610 instr_tx_exec(struct rte_swx_pipeline *p)
1611 {
1612         struct thread *t = &p->threads[p->thread_id];
1613         struct instruction *ip = t->ip;
1614
1615         __instr_tx_exec(p, t, ip);
1616
1617         /* Thread. */
1618         thread_ip_reset(p, t);
1619         instr_rx_exec(p);
1620 }
1621
1622 static inline void
1623 instr_tx_i_exec(struct rte_swx_pipeline *p)
1624 {
1625         struct thread *t = &p->threads[p->thread_id];
1626         struct instruction *ip = t->ip;
1627
1628         __instr_tx_i_exec(p, t, ip);
1629
1630         /* Thread. */
1631         thread_ip_reset(p, t);
1632         instr_rx_exec(p);
1633 }
1634
1635 /*
1636  * extract.
1637  */
1638 static int
1639 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1640                             struct action *action,
1641                             char **tokens,
1642                             int n_tokens,
1643                             struct instruction *instr,
1644                             struct instruction_data *data __rte_unused)
1645 {
1646         struct header *h;
1647
1648         CHECK(!action, EINVAL);
1649         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1650
1651         h = header_parse(p, tokens[1]);
1652         CHECK(h, EINVAL);
1653
1654         if (n_tokens == 2) {
1655                 CHECK(!h->st->var_size, EINVAL);
1656
1657                 instr->type = INSTR_HDR_EXTRACT;
1658                 instr->io.hdr.header_id[0] = h->id;
1659                 instr->io.hdr.struct_id[0] = h->struct_id;
1660                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1661         } else {
1662                 struct field *mf;
1663
1664                 CHECK(h->st->var_size, EINVAL);
1665
1666                 mf = metadata_field_parse(p, tokens[2]);
1667                 CHECK(mf, EINVAL);
1668                 CHECK(!mf->var_size, EINVAL);
1669
1670                 instr->type = INSTR_HDR_EXTRACT_M;
1671                 instr->io.io.offset = mf->offset / 8;
1672                 instr->io.io.n_bits = mf->n_bits;
1673                 instr->io.hdr.header_id[0] = h->id;
1674                 instr->io.hdr.struct_id[0] = h->struct_id;
1675                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1676         }
1677
1678         return 0;
1679 }
1680
1681 static int
1682 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1683                               struct action *action,
1684                               char **tokens,
1685                               int n_tokens,
1686                               struct instruction *instr,
1687                               struct instruction_data *data __rte_unused)
1688 {
1689         struct header *h;
1690
1691         CHECK(!action, EINVAL);
1692         CHECK(n_tokens == 2, EINVAL);
1693
1694         h = header_parse(p, tokens[1]);
1695         CHECK(h, EINVAL);
1696         CHECK(!h->st->var_size, EINVAL);
1697
1698         instr->type = INSTR_HDR_LOOKAHEAD;
1699         instr->io.hdr.header_id[0] = h->id;
1700         instr->io.hdr.struct_id[0] = h->struct_id;
1701         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1702
1703         return 0;
1704 }
1705
1706 static inline void
1707 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1708 {
1709         struct thread *t = &p->threads[p->thread_id];
1710         struct instruction *ip = t->ip;
1711
1712         __instr_hdr_extract_exec(p, t, ip);
1713
1714         /* Thread. */
1715         thread_ip_inc(p);
1716 }
1717
1718 static inline void
1719 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1720 {
1721         struct thread *t = &p->threads[p->thread_id];
1722         struct instruction *ip = t->ip;
1723
1724         __instr_hdr_extract2_exec(p, t, ip);
1725
1726         /* Thread. */
1727         thread_ip_inc(p);
1728 }
1729
1730 static inline void
1731 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1732 {
1733         struct thread *t = &p->threads[p->thread_id];
1734         struct instruction *ip = t->ip;
1735
1736         __instr_hdr_extract3_exec(p, t, ip);
1737
1738         /* Thread. */
1739         thread_ip_inc(p);
1740 }
1741
1742 static inline void
1743 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1744 {
1745         struct thread *t = &p->threads[p->thread_id];
1746         struct instruction *ip = t->ip;
1747
1748         __instr_hdr_extract4_exec(p, t, ip);
1749
1750         /* Thread. */
1751         thread_ip_inc(p);
1752 }
1753
1754 static inline void
1755 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1756 {
1757         struct thread *t = &p->threads[p->thread_id];
1758         struct instruction *ip = t->ip;
1759
1760         __instr_hdr_extract5_exec(p, t, ip);
1761
1762         /* Thread. */
1763         thread_ip_inc(p);
1764 }
1765
1766 static inline void
1767 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1768 {
1769         struct thread *t = &p->threads[p->thread_id];
1770         struct instruction *ip = t->ip;
1771
1772         __instr_hdr_extract6_exec(p, t, ip);
1773
1774         /* Thread. */
1775         thread_ip_inc(p);
1776 }
1777
1778 static inline void
1779 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1780 {
1781         struct thread *t = &p->threads[p->thread_id];
1782         struct instruction *ip = t->ip;
1783
1784         __instr_hdr_extract7_exec(p, t, ip);
1785
1786         /* Thread. */
1787         thread_ip_inc(p);
1788 }
1789
1790 static inline void
1791 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1792 {
1793         struct thread *t = &p->threads[p->thread_id];
1794         struct instruction *ip = t->ip;
1795
1796         __instr_hdr_extract8_exec(p, t, ip);
1797
1798         /* Thread. */
1799         thread_ip_inc(p);
1800 }
1801
1802 static inline void
1803 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1804 {
1805         struct thread *t = &p->threads[p->thread_id];
1806         struct instruction *ip = t->ip;
1807
1808         __instr_hdr_extract_m_exec(p, t, ip);
1809
1810         /* Thread. */
1811         thread_ip_inc(p);
1812 }
1813
1814 static inline void
1815 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1816 {
1817         struct thread *t = &p->threads[p->thread_id];
1818         struct instruction *ip = t->ip;
1819
1820         __instr_hdr_lookahead_exec(p, t, ip);
1821
1822         /* Thread. */
1823         thread_ip_inc(p);
1824 }
1825
1826 /*
1827  * emit.
1828  */
1829 static int
1830 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1831                          struct action *action __rte_unused,
1832                          char **tokens,
1833                          int n_tokens,
1834                          struct instruction *instr,
1835                          struct instruction_data *data __rte_unused)
1836 {
1837         struct header *h;
1838
1839         CHECK(n_tokens == 2, EINVAL);
1840
1841         h = header_parse(p, tokens[1]);
1842         CHECK(h, EINVAL);
1843
1844         instr->type = INSTR_HDR_EMIT;
1845         instr->io.hdr.header_id[0] = h->id;
1846         instr->io.hdr.struct_id[0] = h->struct_id;
1847         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1848         return 0;
1849 }
1850
1851 static inline void
1852 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1853 {
1854         struct thread *t = &p->threads[p->thread_id];
1855         struct instruction *ip = t->ip;
1856
1857         __instr_hdr_emit_exec(p, t, ip);
1858
1859         /* Thread. */
1860         thread_ip_inc(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit2_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit3_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit4_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit5_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 static inline void
1929 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1930 {
1931         struct thread *t = &p->threads[p->thread_id];
1932         struct instruction *ip = t->ip;
1933
1934         __instr_hdr_emit6_tx_exec(p, t, ip);
1935
1936         /* Thread. */
1937         thread_ip_reset(p, t);
1938         instr_rx_exec(p);
1939 }
1940
1941 static inline void
1942 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1943 {
1944         struct thread *t = &p->threads[p->thread_id];
1945         struct instruction *ip = t->ip;
1946
1947         __instr_hdr_emit7_tx_exec(p, t, ip);
1948
1949         /* Thread. */
1950         thread_ip_reset(p, t);
1951         instr_rx_exec(p);
1952 }
1953
1954 static inline void
1955 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1956 {
1957         struct thread *t = &p->threads[p->thread_id];
1958         struct instruction *ip = t->ip;
1959
1960         __instr_hdr_emit8_tx_exec(p, t, ip);
1961
1962         /* Thread. */
1963         thread_ip_reset(p, t);
1964         instr_rx_exec(p);
1965 }
1966
1967 /*
1968  * validate.
1969  */
1970 static int
1971 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1972                              struct action *action __rte_unused,
1973                              char **tokens,
1974                              int n_tokens,
1975                              struct instruction *instr,
1976                              struct instruction_data *data __rte_unused)
1977 {
1978         struct header *h;
1979
1980         CHECK(n_tokens == 2, EINVAL);
1981
1982         h = header_parse(p, tokens[1]);
1983         CHECK(h, EINVAL);
1984
1985         instr->type = INSTR_HDR_VALIDATE;
1986         instr->valid.header_id = h->id;
1987         return 0;
1988 }
1989
1990 static inline void
1991 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1992 {
1993         struct thread *t = &p->threads[p->thread_id];
1994         struct instruction *ip = t->ip;
1995
1996         __instr_hdr_validate_exec(p, t, ip);
1997
1998         /* Thread. */
1999         thread_ip_inc(p);
2000 }
2001
2002 /*
2003  * invalidate.
2004  */
2005 static int
2006 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2007                                struct action *action __rte_unused,
2008                                char **tokens,
2009                                int n_tokens,
2010                                struct instruction *instr,
2011                                struct instruction_data *data __rte_unused)
2012 {
2013         struct header *h;
2014
2015         CHECK(n_tokens == 2, EINVAL);
2016
2017         h = header_parse(p, tokens[1]);
2018         CHECK(h, EINVAL);
2019
2020         instr->type = INSTR_HDR_INVALIDATE;
2021         instr->valid.header_id = h->id;
2022         return 0;
2023 }
2024
2025 static inline void
2026 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2027 {
2028         struct thread *t = &p->threads[p->thread_id];
2029         struct instruction *ip = t->ip;
2030
2031         __instr_hdr_invalidate_exec(p, t, ip);
2032
2033         /* Thread. */
2034         thread_ip_inc(p);
2035 }
2036
2037 /*
2038  * table.
2039  */
2040 static struct table *
2041 table_find(struct rte_swx_pipeline *p, const char *name);
2042
2043 static struct selector *
2044 selector_find(struct rte_swx_pipeline *p, const char *name);
2045
2046 static struct learner *
2047 learner_find(struct rte_swx_pipeline *p, const char *name);
2048
2049 static int
2050 instr_table_translate(struct rte_swx_pipeline *p,
2051                       struct action *action,
2052                       char **tokens,
2053                       int n_tokens,
2054                       struct instruction *instr,
2055                       struct instruction_data *data __rte_unused)
2056 {
2057         struct table *t;
2058         struct selector *s;
2059         struct learner *l;
2060
2061         CHECK(!action, EINVAL);
2062         CHECK(n_tokens == 2, EINVAL);
2063
2064         t = table_find(p, tokens[1]);
2065         if (t) {
2066                 instr->type = INSTR_TABLE;
2067                 instr->table.table_id = t->id;
2068                 return 0;
2069         }
2070
2071         s = selector_find(p, tokens[1]);
2072         if (s) {
2073                 instr->type = INSTR_SELECTOR;
2074                 instr->table.table_id = s->id;
2075                 return 0;
2076         }
2077
2078         l = learner_find(p, tokens[1]);
2079         if (l) {
2080                 instr->type = INSTR_LEARNER;
2081                 instr->table.table_id = l->id;
2082                 return 0;
2083         }
2084
2085         CHECK(0, EINVAL);
2086 }
2087
2088 static inline void
2089 instr_table_exec(struct rte_swx_pipeline *p)
2090 {
2091         struct thread *t = &p->threads[p->thread_id];
2092         struct instruction *ip = t->ip;
2093         uint32_t table_id = ip->table.table_id;
2094         struct rte_swx_table_state *ts = &t->table_state[table_id];
2095         struct table_runtime *table = &t->tables[table_id];
2096         struct table_statistics *stats = &p->table_stats[table_id];
2097         uint64_t action_id, n_pkts_hit, n_pkts_action;
2098         uint8_t *action_data;
2099         int done, hit;
2100
2101         /* Table. */
2102         done = table->func(ts->obj,
2103                            table->mailbox,
2104                            table->key,
2105                            &action_id,
2106                            &action_data,
2107                            &hit);
2108         if (!done) {
2109                 /* Thread. */
2110                 TRACE("[Thread %2u] table %u (not finalized)\n",
2111                       p->thread_id,
2112                       table_id);
2113
2114                 thread_yield(p);
2115                 return;
2116         }
2117
2118         action_id = hit ? action_id : ts->default_action_id;
2119         action_data = hit ? action_data : ts->default_action_data;
2120         n_pkts_hit = stats->n_pkts_hit[hit];
2121         n_pkts_action = stats->n_pkts_action[action_id];
2122
2123         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2124               p->thread_id,
2125               table_id,
2126               hit ? "hit" : "miss",
2127               (uint32_t)action_id);
2128
2129         t->action_id = action_id;
2130         t->structs[0] = action_data;
2131         t->hit = hit;
2132         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2133         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2134
2135         /* Thread. */
2136         thread_ip_action_call(p, t, action_id);
2137 }
2138
2139 static inline void
2140 instr_table_af_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t table_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[table_id];
2146         struct table_runtime *table = &t->tables[table_id];
2147         struct table_statistics *stats = &p->table_stats[table_id];
2148         uint64_t action_id, n_pkts_hit, n_pkts_action;
2149         uint8_t *action_data;
2150         action_func_t action_func;
2151         int done, hit;
2152
2153         /* Table. */
2154         done = table->func(ts->obj,
2155                            table->mailbox,
2156                            table->key,
2157                            &action_id,
2158                            &action_data,
2159                            &hit);
2160         if (!done) {
2161                 /* Thread. */
2162                 TRACE("[Thread %2u] table %u (not finalized)\n",
2163                       p->thread_id,
2164                       table_id);
2165
2166                 thread_yield(p);
2167                 return;
2168         }
2169
2170         action_id = hit ? action_id : ts->default_action_id;
2171         action_data = hit ? action_data : ts->default_action_data;
2172         action_func = p->action_funcs[action_id];
2173         n_pkts_hit = stats->n_pkts_hit[hit];
2174         n_pkts_action = stats->n_pkts_action[action_id];
2175
2176         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2177               p->thread_id,
2178               table_id,
2179               hit ? "hit" : "miss",
2180               (uint32_t)action_id);
2181
2182         t->action_id = action_id;
2183         t->structs[0] = action_data;
2184         t->hit = hit;
2185         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2186         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2187
2188         /* Thread. */
2189         thread_ip_inc(p);
2190
2191         /* Action. */
2192         action_func(p);
2193 }
2194
2195 static inline void
2196 instr_selector_exec(struct rte_swx_pipeline *p)
2197 {
2198         struct thread *t = &p->threads[p->thread_id];
2199         struct instruction *ip = t->ip;
2200         uint32_t selector_id = ip->table.table_id;
2201         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2202         struct selector_runtime *selector = &t->selectors[selector_id];
2203         struct selector_statistics *stats = &p->selector_stats[selector_id];
2204         uint64_t n_pkts = stats->n_pkts;
2205         int done;
2206
2207         /* Table. */
2208         done = rte_swx_table_selector_select(ts->obj,
2209                            selector->mailbox,
2210                            selector->group_id_buffer,
2211                            selector->selector_buffer,
2212                            selector->member_id_buffer);
2213         if (!done) {
2214                 /* Thread. */
2215                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2216                       p->thread_id,
2217                       selector_id);
2218
2219                 thread_yield(p);
2220                 return;
2221         }
2222
2223
2224         TRACE("[Thread %2u] selector %u\n",
2225               p->thread_id,
2226               selector_id);
2227
2228         stats->n_pkts = n_pkts + 1;
2229
2230         /* Thread. */
2231         thread_ip_inc(p);
2232 }
2233
2234 static inline void
2235 instr_learner_exec(struct rte_swx_pipeline *p)
2236 {
2237         struct thread *t = &p->threads[p->thread_id];
2238         struct instruction *ip = t->ip;
2239         uint32_t learner_id = ip->table.table_id;
2240         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2241                 p->n_selectors + learner_id];
2242         struct learner_runtime *l = &t->learners[learner_id];
2243         struct learner_statistics *stats = &p->learner_stats[learner_id];
2244         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2245         uint8_t *action_data;
2246         int done, hit;
2247
2248         /* Table. */
2249         time = rte_get_tsc_cycles();
2250
2251         done = rte_swx_table_learner_lookup(ts->obj,
2252                                             l->mailbox,
2253                                             time,
2254                                             l->key,
2255                                             &action_id,
2256                                             &action_data,
2257                                             &hit);
2258         if (!done) {
2259                 /* Thread. */
2260                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2261                       p->thread_id,
2262                       learner_id);
2263
2264                 thread_yield(p);
2265                 return;
2266         }
2267
2268         action_id = hit ? action_id : ts->default_action_id;
2269         action_data = hit ? action_data : ts->default_action_data;
2270         n_pkts_hit = stats->n_pkts_hit[hit];
2271         n_pkts_action = stats->n_pkts_action[action_id];
2272
2273         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2274               p->thread_id,
2275               learner_id,
2276               hit ? "hit" : "miss",
2277               (uint32_t)action_id);
2278
2279         t->action_id = action_id;
2280         t->structs[0] = action_data;
2281         t->hit = hit;
2282         t->learner_id = learner_id;
2283         t->time = time;
2284         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2285         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2286
2287         /* Thread. */
2288         thread_ip_action_call(p, t, action_id);
2289 }
2290
2291 static inline void
2292 instr_learner_af_exec(struct rte_swx_pipeline *p)
2293 {
2294         struct thread *t = &p->threads[p->thread_id];
2295         struct instruction *ip = t->ip;
2296         uint32_t learner_id = ip->table.table_id;
2297         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2298                 p->n_selectors + learner_id];
2299         struct learner_runtime *l = &t->learners[learner_id];
2300         struct learner_statistics *stats = &p->learner_stats[learner_id];
2301         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2302         uint8_t *action_data;
2303         action_func_t action_func;
2304         int done, hit;
2305
2306         /* Table. */
2307         time = rte_get_tsc_cycles();
2308
2309         done = rte_swx_table_learner_lookup(ts->obj,
2310                                             l->mailbox,
2311                                             time,
2312                                             l->key,
2313                                             &action_id,
2314                                             &action_data,
2315                                             &hit);
2316         if (!done) {
2317                 /* Thread. */
2318                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2319                       p->thread_id,
2320                       learner_id);
2321
2322                 thread_yield(p);
2323                 return;
2324         }
2325
2326         action_id = hit ? action_id : ts->default_action_id;
2327         action_data = hit ? action_data : ts->default_action_data;
2328         action_func = p->action_funcs[action_id];
2329         n_pkts_hit = stats->n_pkts_hit[hit];
2330         n_pkts_action = stats->n_pkts_action[action_id];
2331
2332         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2333               p->thread_id,
2334               learner_id,
2335               hit ? "hit" : "miss",
2336               (uint32_t)action_id);
2337
2338         t->action_id = action_id;
2339         t->structs[0] = action_data;
2340         t->hit = hit;
2341         t->learner_id = learner_id;
2342         t->time = time;
2343         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2344         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2345
2346         /* Thread. */
2347         thread_ip_action_call(p, t, action_id);
2348
2349         /* Action */
2350         action_func(p);
2351 }
2352
2353 /*
2354  * learn.
2355  */
2356 static struct action *
2357 action_find(struct rte_swx_pipeline *p, const char *name);
2358
2359 static int
2360 action_has_nbo_args(struct action *a);
2361
2362 static int
2363 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2364
2365 static int
2366 instr_learn_translate(struct rte_swx_pipeline *p,
2367                       struct action *action,
2368                       char **tokens,
2369                       int n_tokens,
2370                       struct instruction *instr,
2371                       struct instruction_data *data __rte_unused)
2372 {
2373         struct action *a;
2374         const char *mf_name;
2375         uint32_t mf_offset = 0;
2376
2377         CHECK(action, EINVAL);
2378         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2379
2380         a = action_find(p, tokens[1]);
2381         CHECK(a, EINVAL);
2382         CHECK(!action_has_nbo_args(a), EINVAL);
2383
2384         mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2385         CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2386
2387         if (mf_name) {
2388                 struct field *mf;
2389
2390                 mf = metadata_field_parse(p, mf_name);
2391                 CHECK(mf, EINVAL);
2392
2393                 mf_offset = mf->offset / 8;
2394         }
2395
2396         instr->type = INSTR_LEARNER_LEARN;
2397         instr->learn.action_id = a->id;
2398         instr->learn.mf_offset = mf_offset;
2399
2400         return 0;
2401 }
2402
2403 static inline void
2404 instr_learn_exec(struct rte_swx_pipeline *p)
2405 {
2406         struct thread *t = &p->threads[p->thread_id];
2407         struct instruction *ip = t->ip;
2408
2409         __instr_learn_exec(p, t, ip);
2410
2411         /* Thread. */
2412         thread_ip_inc(p);
2413 }
2414
2415 /*
2416  * forget.
2417  */
2418 static int
2419 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2420                        struct action *action,
2421                        char **tokens __rte_unused,
2422                        int n_tokens,
2423                        struct instruction *instr,
2424                        struct instruction_data *data __rte_unused)
2425 {
2426         CHECK(action, EINVAL);
2427         CHECK(n_tokens == 1, EINVAL);
2428
2429         instr->type = INSTR_LEARNER_FORGET;
2430
2431         return 0;
2432 }
2433
2434 static inline void
2435 instr_forget_exec(struct rte_swx_pipeline *p)
2436 {
2437         struct thread *t = &p->threads[p->thread_id];
2438         struct instruction *ip = t->ip;
2439
2440         __instr_forget_exec(p, t, ip);
2441
2442         /* Thread. */
2443         thread_ip_inc(p);
2444 }
2445
2446 /*
2447  * extern.
2448  */
2449 static int
2450 instr_extern_translate(struct rte_swx_pipeline *p,
2451                        struct action *action __rte_unused,
2452                        char **tokens,
2453                        int n_tokens,
2454                        struct instruction *instr,
2455                        struct instruction_data *data __rte_unused)
2456 {
2457         char *token = tokens[1];
2458
2459         CHECK(n_tokens == 2, EINVAL);
2460
2461         if (token[0] == 'e') {
2462                 struct extern_obj *obj;
2463                 struct extern_type_member_func *func;
2464
2465                 func = extern_obj_member_func_parse(p, token, &obj);
2466                 CHECK(func, EINVAL);
2467
2468                 instr->type = INSTR_EXTERN_OBJ;
2469                 instr->ext_obj.ext_obj_id = obj->id;
2470                 instr->ext_obj.func_id = func->id;
2471
2472                 return 0;
2473         }
2474
2475         if (token[0] == 'f') {
2476                 struct extern_func *func;
2477
2478                 func = extern_func_parse(p, token);
2479                 CHECK(func, EINVAL);
2480
2481                 instr->type = INSTR_EXTERN_FUNC;
2482                 instr->ext_func.ext_func_id = func->id;
2483
2484                 return 0;
2485         }
2486
2487         CHECK(0, EINVAL);
2488 }
2489
2490 static inline void
2491 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2492 {
2493         struct thread *t = &p->threads[p->thread_id];
2494         struct instruction *ip = t->ip;
2495         uint32_t done;
2496
2497         /* Extern object member function execute. */
2498         done = __instr_extern_obj_exec(p, t, ip);
2499
2500         /* Thread. */
2501         thread_ip_inc_cond(t, done);
2502         thread_yield_cond(p, done ^ 1);
2503 }
2504
2505 static inline void
2506 instr_extern_func_exec(struct rte_swx_pipeline *p)
2507 {
2508         struct thread *t = &p->threads[p->thread_id];
2509         struct instruction *ip = t->ip;
2510         uint32_t done;
2511
2512         /* Extern function execute. */
2513         done = __instr_extern_func_exec(p, t, ip);
2514
2515         /* Thread. */
2516         thread_ip_inc_cond(t, done);
2517         thread_yield_cond(p, done ^ 1);
2518 }
2519
2520 /*
2521  * mov.
2522  */
2523 static int
2524 instr_mov_translate(struct rte_swx_pipeline *p,
2525                     struct action *action,
2526                     char **tokens,
2527                     int n_tokens,
2528                     struct instruction *instr,
2529                     struct instruction_data *data __rte_unused)
2530 {
2531         char *dst = tokens[1], *src = tokens[2];
2532         struct field *fdst, *fsrc;
2533         uint64_t src_val;
2534         uint32_t dst_struct_id = 0, src_struct_id = 0;
2535
2536         CHECK(n_tokens == 3, EINVAL);
2537
2538         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2539         CHECK(fdst, EINVAL);
2540         CHECK(!fdst->var_size, EINVAL);
2541
2542         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2543         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2544         if (fsrc) {
2545                 CHECK(!fsrc->var_size, EINVAL);
2546
2547                 instr->type = INSTR_MOV;
2548                 if (dst[0] != 'h' && src[0] == 'h')
2549                         instr->type = INSTR_MOV_MH;
2550                 if (dst[0] == 'h' && src[0] != 'h')
2551                         instr->type = INSTR_MOV_HM;
2552                 if (dst[0] == 'h' && src[0] == 'h')
2553                         instr->type = INSTR_MOV_HH;
2554
2555                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2556                 instr->mov.dst.n_bits = fdst->n_bits;
2557                 instr->mov.dst.offset = fdst->offset / 8;
2558                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2559                 instr->mov.src.n_bits = fsrc->n_bits;
2560                 instr->mov.src.offset = fsrc->offset / 8;
2561                 return 0;
2562         }
2563
2564         /* MOV_I. */
2565         src_val = strtoull(src, &src, 0);
2566         CHECK(!src[0], EINVAL);
2567
2568         if (dst[0] == 'h')
2569                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2570
2571         instr->type = INSTR_MOV_I;
2572         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2573         instr->mov.dst.n_bits = fdst->n_bits;
2574         instr->mov.dst.offset = fdst->offset / 8;
2575         instr->mov.src_val = src_val;
2576         return 0;
2577 }
2578
2579 static inline void
2580 instr_mov_exec(struct rte_swx_pipeline *p)
2581 {
2582         struct thread *t = &p->threads[p->thread_id];
2583         struct instruction *ip = t->ip;
2584
2585         __instr_mov_exec(p, t, ip);
2586
2587         /* Thread. */
2588         thread_ip_inc(p);
2589 }
2590
2591 static inline void
2592 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2593 {
2594         struct thread *t = &p->threads[p->thread_id];
2595         struct instruction *ip = t->ip;
2596
2597         __instr_mov_mh_exec(p, t, ip);
2598
2599         /* Thread. */
2600         thread_ip_inc(p);
2601 }
2602
2603 static inline void
2604 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2605 {
2606         struct thread *t = &p->threads[p->thread_id];
2607         struct instruction *ip = t->ip;
2608
2609         __instr_mov_hm_exec(p, t, ip);
2610
2611         /* Thread. */
2612         thread_ip_inc(p);
2613 }
2614
2615 static inline void
2616 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2617 {
2618         struct thread *t = &p->threads[p->thread_id];
2619         struct instruction *ip = t->ip;
2620
2621         __instr_mov_hh_exec(p, t, ip);
2622
2623         /* Thread. */
2624         thread_ip_inc(p);
2625 }
2626
2627 static inline void
2628 instr_mov_i_exec(struct rte_swx_pipeline *p)
2629 {
2630         struct thread *t = &p->threads[p->thread_id];
2631         struct instruction *ip = t->ip;
2632
2633         __instr_mov_i_exec(p, t, ip);
2634
2635         /* Thread. */
2636         thread_ip_inc(p);
2637 }
2638
2639 /*
2640  * dma.
2641  */
2642 static inline void
2643 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2644 {
2645         struct thread *t = &p->threads[p->thread_id];
2646         struct instruction *ip = t->ip;
2647
2648         __instr_dma_ht_exec(p, t, ip);
2649
2650         /* Thread. */
2651         thread_ip_inc(p);
2652 }
2653
2654 static inline void
2655 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2656 {
2657         struct thread *t = &p->threads[p->thread_id];
2658         struct instruction *ip = t->ip;
2659
2660         __instr_dma_ht2_exec(p, t, ip);
2661
2662         /* Thread. */
2663         thread_ip_inc(p);
2664 }
2665
2666 static inline void
2667 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2668 {
2669         struct thread *t = &p->threads[p->thread_id];
2670         struct instruction *ip = t->ip;
2671
2672         __instr_dma_ht3_exec(p, t, ip);
2673
2674         /* Thread. */
2675         thread_ip_inc(p);
2676 }
2677
2678 static inline void
2679 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2680 {
2681         struct thread *t = &p->threads[p->thread_id];
2682         struct instruction *ip = t->ip;
2683
2684         __instr_dma_ht4_exec(p, t, ip);
2685
2686         /* Thread. */
2687         thread_ip_inc(p);
2688 }
2689
2690 static inline void
2691 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2692 {
2693         struct thread *t = &p->threads[p->thread_id];
2694         struct instruction *ip = t->ip;
2695
2696         __instr_dma_ht5_exec(p, t, ip);
2697
2698         /* Thread. */
2699         thread_ip_inc(p);
2700 }
2701
2702 static inline void
2703 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2704 {
2705         struct thread *t = &p->threads[p->thread_id];
2706         struct instruction *ip = t->ip;
2707
2708         __instr_dma_ht6_exec(p, t, ip);
2709
2710         /* Thread. */
2711         thread_ip_inc(p);
2712 }
2713
2714 static inline void
2715 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2716 {
2717         struct thread *t = &p->threads[p->thread_id];
2718         struct instruction *ip = t->ip;
2719
2720         __instr_dma_ht7_exec(p, t, ip);
2721
2722         /* Thread. */
2723         thread_ip_inc(p);
2724 }
2725
2726 static inline void
2727 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2728 {
2729         struct thread *t = &p->threads[p->thread_id];
2730         struct instruction *ip = t->ip;
2731
2732         __instr_dma_ht8_exec(p, t, ip);
2733
2734         /* Thread. */
2735         thread_ip_inc(p);
2736 }
2737
2738 /*
2739  * alu.
2740  */
2741 static int
2742 instr_alu_add_translate(struct rte_swx_pipeline *p,
2743                         struct action *action,
2744                         char **tokens,
2745                         int n_tokens,
2746                         struct instruction *instr,
2747                         struct instruction_data *data __rte_unused)
2748 {
2749         char *dst = tokens[1], *src = tokens[2];
2750         struct field *fdst, *fsrc;
2751         uint64_t src_val;
2752         uint32_t dst_struct_id = 0, src_struct_id = 0;
2753
2754         CHECK(n_tokens == 3, EINVAL);
2755
2756         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2757         CHECK(fdst, EINVAL);
2758         CHECK(!fdst->var_size, EINVAL);
2759
2760         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2761         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2762         if (fsrc) {
2763                 CHECK(!fsrc->var_size, EINVAL);
2764
2765                 instr->type = INSTR_ALU_ADD;
2766                 if (dst[0] == 'h' && src[0] != 'h')
2767                         instr->type = INSTR_ALU_ADD_HM;
2768                 if (dst[0] != 'h' && src[0] == 'h')
2769                         instr->type = INSTR_ALU_ADD_MH;
2770                 if (dst[0] == 'h' && src[0] == 'h')
2771                         instr->type = INSTR_ALU_ADD_HH;
2772
2773                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2774                 instr->alu.dst.n_bits = fdst->n_bits;
2775                 instr->alu.dst.offset = fdst->offset / 8;
2776                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2777                 instr->alu.src.n_bits = fsrc->n_bits;
2778                 instr->alu.src.offset = fsrc->offset / 8;
2779                 return 0;
2780         }
2781
2782         /* ADD_MI, ADD_HI. */
2783         src_val = strtoull(src, &src, 0);
2784         CHECK(!src[0], EINVAL);
2785
2786         instr->type = INSTR_ALU_ADD_MI;
2787         if (dst[0] == 'h')
2788                 instr->type = INSTR_ALU_ADD_HI;
2789
2790         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2791         instr->alu.dst.n_bits = fdst->n_bits;
2792         instr->alu.dst.offset = fdst->offset / 8;
2793         instr->alu.src_val = src_val;
2794         return 0;
2795 }
2796
2797 static int
2798 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2799                         struct action *action,
2800                         char **tokens,
2801                         int n_tokens,
2802                         struct instruction *instr,
2803                         struct instruction_data *data __rte_unused)
2804 {
2805         char *dst = tokens[1], *src = tokens[2];
2806         struct field *fdst, *fsrc;
2807         uint64_t src_val;
2808         uint32_t dst_struct_id = 0, src_struct_id = 0;
2809
2810         CHECK(n_tokens == 3, EINVAL);
2811
2812         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2813         CHECK(fdst, EINVAL);
2814         CHECK(!fdst->var_size, EINVAL);
2815
2816         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2817         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2818         if (fsrc) {
2819                 CHECK(!fsrc->var_size, EINVAL);
2820
2821                 instr->type = INSTR_ALU_SUB;
2822                 if (dst[0] == 'h' && src[0] != 'h')
2823                         instr->type = INSTR_ALU_SUB_HM;
2824                 if (dst[0] != 'h' && src[0] == 'h')
2825                         instr->type = INSTR_ALU_SUB_MH;
2826                 if (dst[0] == 'h' && src[0] == 'h')
2827                         instr->type = INSTR_ALU_SUB_HH;
2828
2829                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2830                 instr->alu.dst.n_bits = fdst->n_bits;
2831                 instr->alu.dst.offset = fdst->offset / 8;
2832                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2833                 instr->alu.src.n_bits = fsrc->n_bits;
2834                 instr->alu.src.offset = fsrc->offset / 8;
2835                 return 0;
2836         }
2837
2838         /* SUB_MI, SUB_HI. */
2839         src_val = strtoull(src, &src, 0);
2840         CHECK(!src[0], EINVAL);
2841
2842         instr->type = INSTR_ALU_SUB_MI;
2843         if (dst[0] == 'h')
2844                 instr->type = INSTR_ALU_SUB_HI;
2845
2846         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2847         instr->alu.dst.n_bits = fdst->n_bits;
2848         instr->alu.dst.offset = fdst->offset / 8;
2849         instr->alu.src_val = src_val;
2850         return 0;
2851 }
2852
2853 static int
2854 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2855                           struct action *action __rte_unused,
2856                           char **tokens,
2857                           int n_tokens,
2858                           struct instruction *instr,
2859                           struct instruction_data *data __rte_unused)
2860 {
2861         char *dst = tokens[1], *src = tokens[2];
2862         struct header *hdst, *hsrc;
2863         struct field *fdst, *fsrc;
2864
2865         CHECK(n_tokens == 3, EINVAL);
2866
2867         fdst = header_field_parse(p, dst, &hdst);
2868         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2869         CHECK(!fdst->var_size, EINVAL);
2870
2871         /* CKADD_FIELD. */
2872         fsrc = header_field_parse(p, src, &hsrc);
2873         if (fsrc) {
2874                 CHECK(!fsrc->var_size, EINVAL);
2875
2876                 instr->type = INSTR_ALU_CKADD_FIELD;
2877                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2878                 instr->alu.dst.n_bits = fdst->n_bits;
2879                 instr->alu.dst.offset = fdst->offset / 8;
2880                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2881                 instr->alu.src.n_bits = fsrc->n_bits;
2882                 instr->alu.src.offset = fsrc->offset / 8;
2883                 return 0;
2884         }
2885
2886         /* CKADD_STRUCT, CKADD_STRUCT20. */
2887         hsrc = header_parse(p, src);
2888         CHECK(hsrc, EINVAL);
2889         CHECK(!hsrc->st->var_size, EINVAL);
2890
2891         instr->type = INSTR_ALU_CKADD_STRUCT;
2892         if ((hsrc->st->n_bits / 8) == 20)
2893                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2894
2895         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2896         instr->alu.dst.n_bits = fdst->n_bits;
2897         instr->alu.dst.offset = fdst->offset / 8;
2898         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2899         instr->alu.src.n_bits = hsrc->st->n_bits;
2900         instr->alu.src.offset = 0; /* Unused. */
2901         return 0;
2902 }
2903
2904 static int
2905 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2906                           struct action *action __rte_unused,
2907                           char **tokens,
2908                           int n_tokens,
2909                           struct instruction *instr,
2910                           struct instruction_data *data __rte_unused)
2911 {
2912         char *dst = tokens[1], *src = tokens[2];
2913         struct header *hdst, *hsrc;
2914         struct field *fdst, *fsrc;
2915
2916         CHECK(n_tokens == 3, EINVAL);
2917
2918         fdst = header_field_parse(p, dst, &hdst);
2919         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2920         CHECK(!fdst->var_size, EINVAL);
2921
2922         fsrc = header_field_parse(p, src, &hsrc);
2923         CHECK(fsrc, EINVAL);
2924         CHECK(!fsrc->var_size, EINVAL);
2925
2926         instr->type = INSTR_ALU_CKSUB_FIELD;
2927         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2928         instr->alu.dst.n_bits = fdst->n_bits;
2929         instr->alu.dst.offset = fdst->offset / 8;
2930         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2931         instr->alu.src.n_bits = fsrc->n_bits;
2932         instr->alu.src.offset = fsrc->offset / 8;
2933         return 0;
2934 }
2935
2936 static int
2937 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2938                         struct action *action,
2939                         char **tokens,
2940                         int n_tokens,
2941                         struct instruction *instr,
2942                         struct instruction_data *data __rte_unused)
2943 {
2944         char *dst = tokens[1], *src = tokens[2];
2945         struct field *fdst, *fsrc;
2946         uint64_t src_val;
2947         uint32_t dst_struct_id = 0, src_struct_id = 0;
2948
2949         CHECK(n_tokens == 3, EINVAL);
2950
2951         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2952         CHECK(fdst, EINVAL);
2953         CHECK(!fdst->var_size, EINVAL);
2954
2955         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2956         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2957         if (fsrc) {
2958                 CHECK(!fsrc->var_size, EINVAL);
2959
2960                 instr->type = INSTR_ALU_SHL;
2961                 if (dst[0] == 'h' && src[0] != 'h')
2962                         instr->type = INSTR_ALU_SHL_HM;
2963                 if (dst[0] != 'h' && src[0] == 'h')
2964                         instr->type = INSTR_ALU_SHL_MH;
2965                 if (dst[0] == 'h' && src[0] == 'h')
2966                         instr->type = INSTR_ALU_SHL_HH;
2967
2968                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2969                 instr->alu.dst.n_bits = fdst->n_bits;
2970                 instr->alu.dst.offset = fdst->offset / 8;
2971                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2972                 instr->alu.src.n_bits = fsrc->n_bits;
2973                 instr->alu.src.offset = fsrc->offset / 8;
2974                 return 0;
2975         }
2976
2977         /* SHL_MI, SHL_HI. */
2978         src_val = strtoull(src, &src, 0);
2979         CHECK(!src[0], EINVAL);
2980
2981         instr->type = INSTR_ALU_SHL_MI;
2982         if (dst[0] == 'h')
2983                 instr->type = INSTR_ALU_SHL_HI;
2984
2985         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2986         instr->alu.dst.n_bits = fdst->n_bits;
2987         instr->alu.dst.offset = fdst->offset / 8;
2988         instr->alu.src_val = src_val;
2989         return 0;
2990 }
2991
2992 static int
2993 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2994                         struct action *action,
2995                         char **tokens,
2996                         int n_tokens,
2997                         struct instruction *instr,
2998                         struct instruction_data *data __rte_unused)
2999 {
3000         char *dst = tokens[1], *src = tokens[2];
3001         struct field *fdst, *fsrc;
3002         uint64_t src_val;
3003         uint32_t dst_struct_id = 0, src_struct_id = 0;
3004
3005         CHECK(n_tokens == 3, EINVAL);
3006
3007         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3008         CHECK(fdst, EINVAL);
3009         CHECK(!fdst->var_size, EINVAL);
3010
3011         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3012         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3013         if (fsrc) {
3014                 CHECK(!fsrc->var_size, EINVAL);
3015
3016                 instr->type = INSTR_ALU_SHR;
3017                 if (dst[0] == 'h' && src[0] != 'h')
3018                         instr->type = INSTR_ALU_SHR_HM;
3019                 if (dst[0] != 'h' && src[0] == 'h')
3020                         instr->type = INSTR_ALU_SHR_MH;
3021                 if (dst[0] == 'h' && src[0] == 'h')
3022                         instr->type = INSTR_ALU_SHR_HH;
3023
3024                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3025                 instr->alu.dst.n_bits = fdst->n_bits;
3026                 instr->alu.dst.offset = fdst->offset / 8;
3027                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3028                 instr->alu.src.n_bits = fsrc->n_bits;
3029                 instr->alu.src.offset = fsrc->offset / 8;
3030                 return 0;
3031         }
3032
3033         /* SHR_MI, SHR_HI. */
3034         src_val = strtoull(src, &src, 0);
3035         CHECK(!src[0], EINVAL);
3036
3037         instr->type = INSTR_ALU_SHR_MI;
3038         if (dst[0] == 'h')
3039                 instr->type = INSTR_ALU_SHR_HI;
3040
3041         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3042         instr->alu.dst.n_bits = fdst->n_bits;
3043         instr->alu.dst.offset = fdst->offset / 8;
3044         instr->alu.src_val = src_val;
3045         return 0;
3046 }
3047
3048 static int
3049 instr_alu_and_translate(struct rte_swx_pipeline *p,
3050                         struct action *action,
3051                         char **tokens,
3052                         int n_tokens,
3053                         struct instruction *instr,
3054                         struct instruction_data *data __rte_unused)
3055 {
3056         char *dst = tokens[1], *src = tokens[2];
3057         struct field *fdst, *fsrc;
3058         uint64_t src_val;
3059         uint32_t dst_struct_id = 0, src_struct_id = 0;
3060
3061         CHECK(n_tokens == 3, EINVAL);
3062
3063         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3064         CHECK(fdst, EINVAL);
3065         CHECK(!fdst->var_size, EINVAL);
3066
3067         /* AND, AND_MH, AND_HM, AND_HH. */
3068         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3069         if (fsrc) {
3070                 CHECK(!fsrc->var_size, EINVAL);
3071
3072                 instr->type = INSTR_ALU_AND;
3073                 if (dst[0] != 'h' && src[0] == 'h')
3074                         instr->type = INSTR_ALU_AND_MH;
3075                 if (dst[0] == 'h' && src[0] != 'h')
3076                         instr->type = INSTR_ALU_AND_HM;
3077                 if (dst[0] == 'h' && src[0] == 'h')
3078                         instr->type = INSTR_ALU_AND_HH;
3079
3080                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3081                 instr->alu.dst.n_bits = fdst->n_bits;
3082                 instr->alu.dst.offset = fdst->offset / 8;
3083                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3084                 instr->alu.src.n_bits = fsrc->n_bits;
3085                 instr->alu.src.offset = fsrc->offset / 8;
3086                 return 0;
3087         }
3088
3089         /* AND_I. */
3090         src_val = strtoull(src, &src, 0);
3091         CHECK(!src[0], EINVAL);
3092
3093         if (dst[0] == 'h')
3094                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3095
3096         instr->type = INSTR_ALU_AND_I;
3097         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3098         instr->alu.dst.n_bits = fdst->n_bits;
3099         instr->alu.dst.offset = fdst->offset / 8;
3100         instr->alu.src_val = src_val;
3101         return 0;
3102 }
3103
3104 static int
3105 instr_alu_or_translate(struct rte_swx_pipeline *p,
3106                        struct action *action,
3107                        char **tokens,
3108                        int n_tokens,
3109                        struct instruction *instr,
3110                        struct instruction_data *data __rte_unused)
3111 {
3112         char *dst = tokens[1], *src = tokens[2];
3113         struct field *fdst, *fsrc;
3114         uint64_t src_val;
3115         uint32_t dst_struct_id = 0, src_struct_id = 0;
3116
3117         CHECK(n_tokens == 3, EINVAL);
3118
3119         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3120         CHECK(fdst, EINVAL);
3121         CHECK(!fdst->var_size, EINVAL);
3122
3123         /* OR, OR_MH, OR_HM, OR_HH. */
3124         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3125         if (fsrc) {
3126                 CHECK(!fsrc->var_size, EINVAL);
3127
3128                 instr->type = INSTR_ALU_OR;
3129                 if (dst[0] != 'h' && src[0] == 'h')
3130                         instr->type = INSTR_ALU_OR_MH;
3131                 if (dst[0] == 'h' && src[0] != 'h')
3132                         instr->type = INSTR_ALU_OR_HM;
3133                 if (dst[0] == 'h' && src[0] == 'h')
3134                         instr->type = INSTR_ALU_OR_HH;
3135
3136                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3137                 instr->alu.dst.n_bits = fdst->n_bits;
3138                 instr->alu.dst.offset = fdst->offset / 8;
3139                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3140                 instr->alu.src.n_bits = fsrc->n_bits;
3141                 instr->alu.src.offset = fsrc->offset / 8;
3142                 return 0;
3143         }
3144
3145         /* OR_I. */
3146         src_val = strtoull(src, &src, 0);
3147         CHECK(!src[0], EINVAL);
3148
3149         if (dst[0] == 'h')
3150                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3151
3152         instr->type = INSTR_ALU_OR_I;
3153         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3154         instr->alu.dst.n_bits = fdst->n_bits;
3155         instr->alu.dst.offset = fdst->offset / 8;
3156         instr->alu.src_val = src_val;
3157         return 0;
3158 }
3159
3160 static int
3161 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3162                         struct action *action,
3163                         char **tokens,
3164                         int n_tokens,
3165                         struct instruction *instr,
3166                         struct instruction_data *data __rte_unused)
3167 {
3168         char *dst = tokens[1], *src = tokens[2];
3169         struct field *fdst, *fsrc;
3170         uint64_t src_val;
3171         uint32_t dst_struct_id = 0, src_struct_id = 0;
3172
3173         CHECK(n_tokens == 3, EINVAL);
3174
3175         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3176         CHECK(fdst, EINVAL);
3177         CHECK(!fdst->var_size, EINVAL);
3178
3179         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3180         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3181         if (fsrc) {
3182                 CHECK(!fsrc->var_size, EINVAL);
3183
3184                 instr->type = INSTR_ALU_XOR;
3185                 if (dst[0] != 'h' && src[0] == 'h')
3186                         instr->type = INSTR_ALU_XOR_MH;
3187                 if (dst[0] == 'h' && src[0] != 'h')
3188                         instr->type = INSTR_ALU_XOR_HM;
3189                 if (dst[0] == 'h' && src[0] == 'h')
3190                         instr->type = INSTR_ALU_XOR_HH;
3191
3192                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3193                 instr->alu.dst.n_bits = fdst->n_bits;
3194                 instr->alu.dst.offset = fdst->offset / 8;
3195                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3196                 instr->alu.src.n_bits = fsrc->n_bits;
3197                 instr->alu.src.offset = fsrc->offset / 8;
3198                 return 0;
3199         }
3200
3201         /* XOR_I. */
3202         src_val = strtoull(src, &src, 0);
3203         CHECK(!src[0], EINVAL);
3204
3205         if (dst[0] == 'h')
3206                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3207
3208         instr->type = INSTR_ALU_XOR_I;
3209         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3210         instr->alu.dst.n_bits = fdst->n_bits;
3211         instr->alu.dst.offset = fdst->offset / 8;
3212         instr->alu.src_val = src_val;
3213         return 0;
3214 }
3215
3216 static inline void
3217 instr_alu_add_exec(struct rte_swx_pipeline *p)
3218 {
3219         struct thread *t = &p->threads[p->thread_id];
3220         struct instruction *ip = t->ip;
3221
3222         /* Structs */
3223         __instr_alu_add_exec(p, t, ip);
3224
3225         /* Thread. */
3226         thread_ip_inc(p);
3227 }
3228
3229 static inline void
3230 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3231 {
3232         struct thread *t = &p->threads[p->thread_id];
3233         struct instruction *ip = t->ip;
3234
3235         /* Structs. */
3236         __instr_alu_add_mh_exec(p, t, ip);
3237
3238         /* Thread. */
3239         thread_ip_inc(p);
3240 }
3241
3242 static inline void
3243 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3244 {
3245         struct thread *t = &p->threads[p->thread_id];
3246         struct instruction *ip = t->ip;
3247
3248         /* Structs. */
3249         __instr_alu_add_hm_exec(p, t, ip);
3250
3251         /* Thread. */
3252         thread_ip_inc(p);
3253 }
3254
3255 static inline void
3256 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3257 {
3258         struct thread *t = &p->threads[p->thread_id];
3259         struct instruction *ip = t->ip;
3260
3261         /* Structs. */
3262         __instr_alu_add_hh_exec(p, t, ip);
3263
3264         /* Thread. */
3265         thread_ip_inc(p);
3266 }
3267
3268 static inline void
3269 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3270 {
3271         struct thread *t = &p->threads[p->thread_id];
3272         struct instruction *ip = t->ip;
3273
3274         /* Structs. */
3275         __instr_alu_add_mi_exec(p, t, ip);
3276
3277         /* Thread. */
3278         thread_ip_inc(p);
3279 }
3280
3281 static inline void
3282 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3283 {
3284         struct thread *t = &p->threads[p->thread_id];
3285         struct instruction *ip = t->ip;
3286
3287         /* Structs. */
3288         __instr_alu_add_hi_exec(p, t, ip);
3289
3290         /* Thread. */
3291         thread_ip_inc(p);
3292 }
3293
3294 static inline void
3295 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3296 {
3297         struct thread *t = &p->threads[p->thread_id];
3298         struct instruction *ip = t->ip;
3299
3300         /* Structs. */
3301         __instr_alu_sub_exec(p, t, ip);
3302
3303         /* Thread. */
3304         thread_ip_inc(p);
3305 }
3306
3307 static inline void
3308 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3309 {
3310         struct thread *t = &p->threads[p->thread_id];
3311         struct instruction *ip = t->ip;
3312
3313         /* Structs. */
3314         __instr_alu_sub_mh_exec(p, t, ip);
3315
3316         /* Thread. */
3317         thread_ip_inc(p);
3318 }
3319
3320 static inline void
3321 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3322 {
3323         struct thread *t = &p->threads[p->thread_id];
3324         struct instruction *ip = t->ip;
3325
3326         /* Structs. */
3327         __instr_alu_sub_hm_exec(p, t, ip);
3328
3329         /* Thread. */
3330         thread_ip_inc(p);
3331 }
3332
3333 static inline void
3334 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3335 {
3336         struct thread *t = &p->threads[p->thread_id];
3337         struct instruction *ip = t->ip;
3338
3339         /* Structs. */
3340         __instr_alu_sub_hh_exec(p, t, ip);
3341
3342         /* Thread. */
3343         thread_ip_inc(p);
3344 }
3345
3346 static inline void
3347 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3348 {
3349         struct thread *t = &p->threads[p->thread_id];
3350         struct instruction *ip = t->ip;
3351
3352         /* Structs. */
3353         __instr_alu_sub_mi_exec(p, t, ip);
3354
3355         /* Thread. */
3356         thread_ip_inc(p);
3357 }
3358
3359 static inline void
3360 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3361 {
3362         struct thread *t = &p->threads[p->thread_id];
3363         struct instruction *ip = t->ip;
3364
3365         /* Structs. */
3366         __instr_alu_sub_hi_exec(p, t, ip);
3367
3368         /* Thread. */
3369         thread_ip_inc(p);
3370 }
3371
3372 static inline void
3373 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3374 {
3375         struct thread *t = &p->threads[p->thread_id];
3376         struct instruction *ip = t->ip;
3377
3378         /* Structs. */
3379         __instr_alu_shl_exec(p, t, ip);
3380
3381         /* Thread. */
3382         thread_ip_inc(p);
3383 }
3384
3385 static inline void
3386 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3387 {
3388         struct thread *t = &p->threads[p->thread_id];
3389         struct instruction *ip = t->ip;
3390
3391         /* Structs. */
3392         __instr_alu_shl_mh_exec(p, t, ip);
3393
3394         /* Thread. */
3395         thread_ip_inc(p);
3396 }
3397
3398 static inline void
3399 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3400 {
3401         struct thread *t = &p->threads[p->thread_id];
3402         struct instruction *ip = t->ip;
3403
3404         /* Structs. */
3405         __instr_alu_shl_hm_exec(p, t, ip);
3406
3407         /* Thread. */
3408         thread_ip_inc(p);
3409 }
3410
3411 static inline void
3412 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3413 {
3414         struct thread *t = &p->threads[p->thread_id];
3415         struct instruction *ip = t->ip;
3416
3417         /* Structs. */
3418         __instr_alu_shl_hh_exec(p, t, ip);
3419
3420         /* Thread. */
3421         thread_ip_inc(p);
3422 }
3423
3424 static inline void
3425 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3426 {
3427         struct thread *t = &p->threads[p->thread_id];
3428         struct instruction *ip = t->ip;
3429
3430         /* Structs. */
3431         __instr_alu_shl_mi_exec(p, t, ip);
3432
3433         /* Thread. */
3434         thread_ip_inc(p);
3435 }
3436
3437 static inline void
3438 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3439 {
3440         struct thread *t = &p->threads[p->thread_id];
3441         struct instruction *ip = t->ip;
3442
3443         /* Structs. */
3444         __instr_alu_shl_hi_exec(p, t, ip);
3445
3446         /* Thread. */
3447         thread_ip_inc(p);
3448 }
3449
3450 static inline void
3451 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3452 {
3453         struct thread *t = &p->threads[p->thread_id];
3454         struct instruction *ip = t->ip;
3455
3456         /* Structs. */
3457         __instr_alu_shr_exec(p, t, ip);
3458
3459         /* Thread. */
3460         thread_ip_inc(p);
3461 }
3462
3463 static inline void
3464 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3465 {
3466         struct thread *t = &p->threads[p->thread_id];
3467         struct instruction *ip = t->ip;
3468
3469         /* Structs. */
3470         __instr_alu_shr_mh_exec(p, t, ip);
3471
3472         /* Thread. */
3473         thread_ip_inc(p);
3474 }
3475
3476 static inline void
3477 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3478 {
3479         struct thread *t = &p->threads[p->thread_id];
3480         struct instruction *ip = t->ip;
3481
3482         /* Structs. */
3483         __instr_alu_shr_hm_exec(p, t, ip);
3484
3485         /* Thread. */
3486         thread_ip_inc(p);
3487 }
3488
3489 static inline void
3490 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3491 {
3492         struct thread *t = &p->threads[p->thread_id];
3493         struct instruction *ip = t->ip;
3494
3495         /* Structs. */
3496         __instr_alu_shr_hh_exec(p, t, ip);
3497
3498         /* Thread. */
3499         thread_ip_inc(p);
3500 }
3501
3502 static inline void
3503 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3504 {
3505         struct thread *t = &p->threads[p->thread_id];
3506         struct instruction *ip = t->ip;
3507
3508         /* Structs. */
3509         __instr_alu_shr_mi_exec(p, t, ip);
3510
3511         /* Thread. */
3512         thread_ip_inc(p);
3513 }
3514
3515 static inline void
3516 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3517 {
3518         struct thread *t = &p->threads[p->thread_id];
3519         struct instruction *ip = t->ip;
3520
3521         /* Structs. */
3522         __instr_alu_shr_hi_exec(p, t, ip);
3523
3524         /* Thread. */
3525         thread_ip_inc(p);
3526 }
3527
3528 static inline void
3529 instr_alu_and_exec(struct rte_swx_pipeline *p)
3530 {
3531         struct thread *t = &p->threads[p->thread_id];
3532         struct instruction *ip = t->ip;
3533
3534         /* Structs. */
3535         __instr_alu_and_exec(p, t, ip);
3536
3537         /* Thread. */
3538         thread_ip_inc(p);
3539 }
3540
3541 static inline void
3542 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3543 {
3544         struct thread *t = &p->threads[p->thread_id];
3545         struct instruction *ip = t->ip;
3546
3547         /* Structs. */
3548         __instr_alu_and_mh_exec(p, t, ip);
3549
3550         /* Thread. */
3551         thread_ip_inc(p);
3552 }
3553
3554 static inline void
3555 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3556 {
3557         struct thread *t = &p->threads[p->thread_id];
3558         struct instruction *ip = t->ip;
3559
3560         /* Structs. */
3561         __instr_alu_and_hm_exec(p, t, ip);
3562
3563         /* Thread. */
3564         thread_ip_inc(p);
3565 }
3566
3567 static inline void
3568 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3569 {
3570         struct thread *t = &p->threads[p->thread_id];
3571         struct instruction *ip = t->ip;
3572
3573         /* Structs. */
3574         __instr_alu_and_hh_exec(p, t, ip);
3575
3576         /* Thread. */
3577         thread_ip_inc(p);
3578 }
3579
3580 static inline void
3581 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3582 {
3583         struct thread *t = &p->threads[p->thread_id];
3584         struct instruction *ip = t->ip;
3585
3586         /* Structs. */
3587         __instr_alu_and_i_exec(p, t, ip);
3588
3589         /* Thread. */
3590         thread_ip_inc(p);
3591 }
3592
3593 static inline void
3594 instr_alu_or_exec(struct rte_swx_pipeline *p)
3595 {
3596         struct thread *t = &p->threads[p->thread_id];
3597         struct instruction *ip = t->ip;
3598
3599         /* Structs. */
3600         __instr_alu_or_exec(p, t, ip);
3601
3602         /* Thread. */
3603         thread_ip_inc(p);
3604 }
3605
3606 static inline void
3607 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3608 {
3609         struct thread *t = &p->threads[p->thread_id];
3610         struct instruction *ip = t->ip;
3611
3612         /* Structs. */
3613         __instr_alu_or_mh_exec(p, t, ip);
3614
3615         /* Thread. */
3616         thread_ip_inc(p);
3617 }
3618
3619 static inline void
3620 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3621 {
3622         struct thread *t = &p->threads[p->thread_id];
3623         struct instruction *ip = t->ip;
3624
3625         /* Structs. */
3626         __instr_alu_or_hm_exec(p, t, ip);
3627
3628         /* Thread. */
3629         thread_ip_inc(p);
3630 }
3631
3632 static inline void
3633 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3634 {
3635         struct thread *t = &p->threads[p->thread_id];
3636         struct instruction *ip = t->ip;
3637
3638         /* Structs. */
3639         __instr_alu_or_hh_exec(p, t, ip);
3640
3641         /* Thread. */
3642         thread_ip_inc(p);
3643 }
3644
3645 static inline void
3646 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3647 {
3648         struct thread *t = &p->threads[p->thread_id];
3649         struct instruction *ip = t->ip;
3650
3651         /* Structs. */
3652         __instr_alu_or_i_exec(p, t, ip);
3653
3654         /* Thread. */
3655         thread_ip_inc(p);
3656 }
3657
3658 static inline void
3659 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3660 {
3661         struct thread *t = &p->threads[p->thread_id];
3662         struct instruction *ip = t->ip;
3663
3664         /* Structs. */
3665         __instr_alu_xor_exec(p, t, ip);
3666
3667         /* Thread. */
3668         thread_ip_inc(p);
3669 }
3670
3671 static inline void
3672 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3673 {
3674         struct thread *t = &p->threads[p->thread_id];
3675         struct instruction *ip = t->ip;
3676
3677         /* Structs. */
3678         __instr_alu_xor_mh_exec(p, t, ip);
3679
3680         /* Thread. */
3681         thread_ip_inc(p);
3682 }
3683
3684 static inline void
3685 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3686 {
3687         struct thread *t = &p->threads[p->thread_id];
3688         struct instruction *ip = t->ip;
3689
3690         /* Structs. */
3691         __instr_alu_xor_hm_exec(p, t, ip);
3692
3693         /* Thread. */
3694         thread_ip_inc(p);
3695 }
3696
3697 static inline void
3698 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3699 {
3700         struct thread *t = &p->threads[p->thread_id];
3701         struct instruction *ip = t->ip;
3702
3703         /* Structs. */
3704         __instr_alu_xor_hh_exec(p, t, ip);
3705
3706         /* Thread. */
3707         thread_ip_inc(p);
3708 }
3709
3710 static inline void
3711 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3712 {
3713         struct thread *t = &p->threads[p->thread_id];
3714         struct instruction *ip = t->ip;
3715
3716         /* Structs. */
3717         __instr_alu_xor_i_exec(p, t, ip);
3718
3719         /* Thread. */
3720         thread_ip_inc(p);
3721 }
3722
3723 static inline void
3724 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3725 {
3726         struct thread *t = &p->threads[p->thread_id];
3727         struct instruction *ip = t->ip;
3728
3729         /* Structs. */
3730         __instr_alu_ckadd_field_exec(p, t, ip);
3731
3732         /* Thread. */
3733         thread_ip_inc(p);
3734 }
3735
3736 static inline void
3737 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3738 {
3739         struct thread *t = &p->threads[p->thread_id];
3740         struct instruction *ip = t->ip;
3741
3742         /* Structs. */
3743         __instr_alu_cksub_field_exec(p, t, ip);
3744
3745         /* Thread. */
3746         thread_ip_inc(p);
3747 }
3748
3749 static inline void
3750 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3751 {
3752         struct thread *t = &p->threads[p->thread_id];
3753         struct instruction *ip = t->ip;
3754
3755         /* Structs. */
3756         __instr_alu_ckadd_struct20_exec(p, t, ip);
3757
3758         /* Thread. */
3759         thread_ip_inc(p);
3760 }
3761
3762 static inline void
3763 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3764 {
3765         struct thread *t = &p->threads[p->thread_id];
3766         struct instruction *ip = t->ip;
3767
3768         /* Structs. */
3769         __instr_alu_ckadd_struct_exec(p, t, ip);
3770
3771         /* Thread. */
3772         thread_ip_inc(p);
3773 }
3774
3775 /*
3776  * Register array.
3777  */
3778 static struct regarray *
3779 regarray_find(struct rte_swx_pipeline *p, const char *name);
3780
3781 static int
3782 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3783                       struct action *action,
3784                       char **tokens,
3785                       int n_tokens,
3786                       struct instruction *instr,
3787                       struct instruction_data *data __rte_unused)
3788 {
3789         char *regarray = tokens[1], *idx = tokens[2];
3790         struct regarray *r;
3791         struct field *fidx;
3792         uint32_t idx_struct_id, idx_val;
3793
3794         CHECK(n_tokens == 3, EINVAL);
3795
3796         r = regarray_find(p, regarray);
3797         CHECK(r, EINVAL);
3798
3799         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3800         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3801         if (fidx) {
3802                 CHECK(!fidx->var_size, EINVAL);
3803
3804                 instr->type = INSTR_REGPREFETCH_RM;
3805                 if (idx[0] == 'h')
3806                         instr->type = INSTR_REGPREFETCH_RH;
3807
3808                 instr->regarray.regarray_id = r->id;
3809                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3810                 instr->regarray.idx.n_bits = fidx->n_bits;
3811                 instr->regarray.idx.offset = fidx->offset / 8;
3812                 instr->regarray.dstsrc_val = 0; /* Unused. */
3813                 return 0;
3814         }
3815
3816         /* REGPREFETCH_RI. */
3817         idx_val = strtoul(idx, &idx, 0);
3818         CHECK(!idx[0], EINVAL);
3819
3820         instr->type = INSTR_REGPREFETCH_RI;
3821         instr->regarray.regarray_id = r->id;
3822         instr->regarray.idx_val = idx_val;
3823         instr->regarray.dstsrc_val = 0; /* Unused. */
3824         return 0;
3825 }
3826
3827 static int
3828 instr_regrd_translate(struct rte_swx_pipeline *p,
3829                       struct action *action,
3830                       char **tokens,
3831                       int n_tokens,
3832                       struct instruction *instr,
3833                       struct instruction_data *data __rte_unused)
3834 {
3835         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3836         struct regarray *r;
3837         struct field *fdst, *fidx;
3838         uint32_t dst_struct_id, idx_struct_id, idx_val;
3839
3840         CHECK(n_tokens == 4, EINVAL);
3841
3842         r = regarray_find(p, regarray);
3843         CHECK(r, EINVAL);
3844
3845         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3846         CHECK(fdst, EINVAL);
3847         CHECK(!fdst->var_size, EINVAL);
3848
3849         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3850         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3851         if (fidx) {
3852                 CHECK(!fidx->var_size, EINVAL);
3853
3854                 instr->type = INSTR_REGRD_MRM;
3855                 if (dst[0] == 'h' && idx[0] != 'h')
3856                         instr->type = INSTR_REGRD_HRM;
3857                 if (dst[0] != 'h' && idx[0] == 'h')
3858                         instr->type = INSTR_REGRD_MRH;
3859                 if (dst[0] == 'h' && idx[0] == 'h')
3860                         instr->type = INSTR_REGRD_HRH;
3861
3862                 instr->regarray.regarray_id = r->id;
3863                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3864                 instr->regarray.idx.n_bits = fidx->n_bits;
3865                 instr->regarray.idx.offset = fidx->offset / 8;
3866                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3867                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3868                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3869                 return 0;
3870         }
3871
3872         /* REGRD_MRI, REGRD_HRI. */
3873         idx_val = strtoul(idx, &idx, 0);
3874         CHECK(!idx[0], EINVAL);
3875
3876         instr->type = INSTR_REGRD_MRI;
3877         if (dst[0] == 'h')
3878                 instr->type = INSTR_REGRD_HRI;
3879
3880         instr->regarray.regarray_id = r->id;
3881         instr->regarray.idx_val = idx_val;
3882         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3883         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3884         instr->regarray.dstsrc.offset = fdst->offset / 8;
3885         return 0;
3886 }
3887
3888 static int
3889 instr_regwr_translate(struct rte_swx_pipeline *p,
3890                       struct action *action,
3891                       char **tokens,
3892                       int n_tokens,
3893                       struct instruction *instr,
3894                       struct instruction_data *data __rte_unused)
3895 {
3896         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3897         struct regarray *r;
3898         struct field *fidx, *fsrc;
3899         uint64_t src_val;
3900         uint32_t idx_struct_id, idx_val, src_struct_id;
3901
3902         CHECK(n_tokens == 4, EINVAL);
3903
3904         r = regarray_find(p, regarray);
3905         CHECK(r, EINVAL);
3906
3907         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3908         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3909         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3910         if (fidx && fsrc) {
3911                 CHECK(!fidx->var_size, EINVAL);
3912                 CHECK(!fsrc->var_size, EINVAL);
3913
3914                 instr->type = INSTR_REGWR_RMM;
3915                 if (idx[0] == 'h' && src[0] != 'h')
3916                         instr->type = INSTR_REGWR_RHM;
3917                 if (idx[0] != 'h' && src[0] == 'h')
3918                         instr->type = INSTR_REGWR_RMH;
3919                 if (idx[0] == 'h' && src[0] == 'h')
3920                         instr->type = INSTR_REGWR_RHH;
3921
3922                 instr->regarray.regarray_id = r->id;
3923                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3924                 instr->regarray.idx.n_bits = fidx->n_bits;
3925                 instr->regarray.idx.offset = fidx->offset / 8;
3926                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3927                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3928                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3929                 return 0;
3930         }
3931
3932         /* REGWR_RHI, REGWR_RMI. */
3933         if (fidx && !fsrc) {
3934                 CHECK(!fidx->var_size, EINVAL);
3935
3936                 src_val = strtoull(src, &src, 0);
3937                 CHECK(!src[0], EINVAL);
3938
3939                 instr->type = INSTR_REGWR_RMI;
3940                 if (idx[0] == 'h')
3941                         instr->type = INSTR_REGWR_RHI;
3942
3943                 instr->regarray.regarray_id = r->id;
3944                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3945                 instr->regarray.idx.n_bits = fidx->n_bits;
3946                 instr->regarray.idx.offset = fidx->offset / 8;
3947                 instr->regarray.dstsrc_val = src_val;
3948                 return 0;
3949         }
3950
3951         /* REGWR_RIH, REGWR_RIM. */
3952         if (!fidx && fsrc) {
3953                 idx_val = strtoul(idx, &idx, 0);
3954                 CHECK(!idx[0], EINVAL);
3955
3956                 CHECK(!fsrc->var_size, EINVAL);
3957
3958                 instr->type = INSTR_REGWR_RIM;
3959                 if (src[0] == 'h')
3960                         instr->type = INSTR_REGWR_RIH;
3961
3962                 instr->regarray.regarray_id = r->id;
3963                 instr->regarray.idx_val = idx_val;
3964                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3965                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3966                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3967                 return 0;
3968         }
3969
3970         /* REGWR_RII. */
3971         src_val = strtoull(src, &src, 0);
3972         CHECK(!src[0], EINVAL);
3973
3974         idx_val = strtoul(idx, &idx, 0);
3975         CHECK(!idx[0], EINVAL);
3976
3977         instr->type = INSTR_REGWR_RII;
3978         instr->regarray.idx_val = idx_val;
3979         instr->regarray.dstsrc_val = src_val;
3980
3981         return 0;
3982 }
3983
3984 static int
3985 instr_regadd_translate(struct rte_swx_pipeline *p,
3986                        struct action *action,
3987                        char **tokens,
3988                        int n_tokens,
3989                        struct instruction *instr,
3990                        struct instruction_data *data __rte_unused)
3991 {
3992         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3993         struct regarray *r;
3994         struct field *fidx, *fsrc;
3995         uint64_t src_val;
3996         uint32_t idx_struct_id, idx_val, src_struct_id;
3997
3998         CHECK(n_tokens == 4, EINVAL);
3999
4000         r = regarray_find(p, regarray);
4001         CHECK(r, EINVAL);
4002
4003         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4004         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4005         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4006         if (fidx && fsrc) {
4007                 CHECK(!fidx->var_size, EINVAL);
4008                 CHECK(!fsrc->var_size, EINVAL);
4009
4010                 instr->type = INSTR_REGADD_RMM;
4011                 if (idx[0] == 'h' && src[0] != 'h')
4012                         instr->type = INSTR_REGADD_RHM;
4013                 if (idx[0] != 'h' && src[0] == 'h')
4014                         instr->type = INSTR_REGADD_RMH;
4015                 if (idx[0] == 'h' && src[0] == 'h')
4016                         instr->type = INSTR_REGADD_RHH;
4017
4018                 instr->regarray.regarray_id = r->id;
4019                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4020                 instr->regarray.idx.n_bits = fidx->n_bits;
4021                 instr->regarray.idx.offset = fidx->offset / 8;
4022                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4023                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4024                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4025                 return 0;
4026         }
4027
4028         /* REGADD_RHI, REGADD_RMI. */
4029         if (fidx && !fsrc) {
4030                 CHECK(!fidx->var_size, EINVAL);
4031
4032                 src_val = strtoull(src, &src, 0);
4033                 CHECK(!src[0], EINVAL);
4034
4035                 instr->type = INSTR_REGADD_RMI;
4036                 if (idx[0] == 'h')
4037                         instr->type = INSTR_REGADD_RHI;
4038
4039                 instr->regarray.regarray_id = r->id;
4040                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4041                 instr->regarray.idx.n_bits = fidx->n_bits;
4042                 instr->regarray.idx.offset = fidx->offset / 8;
4043                 instr->regarray.dstsrc_val = src_val;
4044                 return 0;
4045         }
4046
4047         /* REGADD_RIH, REGADD_RIM. */
4048         if (!fidx && fsrc) {
4049                 idx_val = strtoul(idx, &idx, 0);
4050                 CHECK(!idx[0], EINVAL);
4051
4052                 CHECK(!fsrc->var_size, EINVAL);
4053
4054                 instr->type = INSTR_REGADD_RIM;
4055                 if (src[0] == 'h')
4056                         instr->type = INSTR_REGADD_RIH;
4057
4058                 instr->regarray.regarray_id = r->id;
4059                 instr->regarray.idx_val = idx_val;
4060                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4061                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4062                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4063                 return 0;
4064         }
4065
4066         /* REGADD_RII. */
4067         src_val = strtoull(src, &src, 0);
4068         CHECK(!src[0], EINVAL);
4069
4070         idx_val = strtoul(idx, &idx, 0);
4071         CHECK(!idx[0], EINVAL);
4072
4073         instr->type = INSTR_REGADD_RII;
4074         instr->regarray.idx_val = idx_val;
4075         instr->regarray.dstsrc_val = src_val;
4076         return 0;
4077 }
4078
4079 static inline void
4080 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4081 {
4082         struct thread *t = &p->threads[p->thread_id];
4083         struct instruction *ip = t->ip;
4084
4085         /* Structs. */
4086         __instr_regprefetch_rh_exec(p, t, ip);
4087
4088         /* Thread. */
4089         thread_ip_inc(p);
4090 }
4091
4092 static inline void
4093 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4094 {
4095         struct thread *t = &p->threads[p->thread_id];
4096         struct instruction *ip = t->ip;
4097
4098         /* Structs. */
4099         __instr_regprefetch_rm_exec(p, t, ip);
4100
4101         /* Thread. */
4102         thread_ip_inc(p);
4103 }
4104
4105 static inline void
4106 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4107 {
4108         struct thread *t = &p->threads[p->thread_id];
4109         struct instruction *ip = t->ip;
4110
4111         /* Structs. */
4112         __instr_regprefetch_ri_exec(p, t, ip);
4113
4114         /* Thread. */
4115         thread_ip_inc(p);
4116 }
4117
4118 static inline void
4119 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4120 {
4121         struct thread *t = &p->threads[p->thread_id];
4122         struct instruction *ip = t->ip;
4123
4124         /* Structs. */
4125         __instr_regrd_hrh_exec(p, t, ip);
4126
4127         /* Thread. */
4128         thread_ip_inc(p);
4129 }
4130
4131 static inline void
4132 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4133 {
4134         struct thread *t = &p->threads[p->thread_id];
4135         struct instruction *ip = t->ip;
4136
4137         /* Structs. */
4138         __instr_regrd_hrm_exec(p, t, ip);
4139
4140         /* Thread. */
4141         thread_ip_inc(p);
4142 }
4143
4144 static inline void
4145 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4146 {
4147         struct thread *t = &p->threads[p->thread_id];
4148         struct instruction *ip = t->ip;
4149
4150         /* Structs. */
4151         __instr_regrd_mrh_exec(p, t, ip);
4152
4153         /* Thread. */
4154         thread_ip_inc(p);
4155 }
4156
4157 static inline void
4158 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4159 {
4160         struct thread *t = &p->threads[p->thread_id];
4161         struct instruction *ip = t->ip;
4162
4163         /* Structs. */
4164         __instr_regrd_mrm_exec(p, t, ip);
4165
4166         /* Thread. */
4167         thread_ip_inc(p);
4168 }
4169
4170 static inline void
4171 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4172 {
4173         struct thread *t = &p->threads[p->thread_id];
4174         struct instruction *ip = t->ip;
4175
4176         /* Structs. */
4177         __instr_regrd_hri_exec(p, t, ip);
4178
4179         /* Thread. */
4180         thread_ip_inc(p);
4181 }
4182
4183 static inline void
4184 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4185 {
4186         struct thread *t = &p->threads[p->thread_id];
4187         struct instruction *ip = t->ip;
4188
4189         /* Structs. */
4190         __instr_regrd_mri_exec(p, t, ip);
4191
4192         /* Thread. */
4193         thread_ip_inc(p);
4194 }
4195
4196 static inline void
4197 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4198 {
4199         struct thread *t = &p->threads[p->thread_id];
4200         struct instruction *ip = t->ip;
4201
4202         /* Structs. */
4203         __instr_regwr_rhh_exec(p, t, ip);
4204
4205         /* Thread. */
4206         thread_ip_inc(p);
4207 }
4208
4209 static inline void
4210 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4211 {
4212         struct thread *t = &p->threads[p->thread_id];
4213         struct instruction *ip = t->ip;
4214
4215         /* Structs. */
4216         __instr_regwr_rhm_exec(p, t, ip);
4217
4218         /* Thread. */
4219         thread_ip_inc(p);
4220 }
4221
4222 static inline void
4223 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4224 {
4225         struct thread *t = &p->threads[p->thread_id];
4226         struct instruction *ip = t->ip;
4227
4228         /* Structs. */
4229         __instr_regwr_rmh_exec(p, t, ip);
4230
4231         /* Thread. */
4232         thread_ip_inc(p);
4233 }
4234
4235 static inline void
4236 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4237 {
4238         struct thread *t = &p->threads[p->thread_id];
4239         struct instruction *ip = t->ip;
4240
4241         /* Structs. */
4242         __instr_regwr_rmm_exec(p, t, ip);
4243
4244         /* Thread. */
4245         thread_ip_inc(p);
4246 }
4247
4248 static inline void
4249 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4250 {
4251         struct thread *t = &p->threads[p->thread_id];
4252         struct instruction *ip = t->ip;
4253
4254         /* Structs. */
4255         __instr_regwr_rhi_exec(p, t, ip);
4256
4257         /* Thread. */
4258         thread_ip_inc(p);
4259 }
4260
4261 static inline void
4262 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4263 {
4264         struct thread *t = &p->threads[p->thread_id];
4265         struct instruction *ip = t->ip;
4266
4267         /* Structs. */
4268         __instr_regwr_rmi_exec(p, t, ip);
4269
4270         /* Thread. */
4271         thread_ip_inc(p);
4272 }
4273
4274 static inline void
4275 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4276 {
4277         struct thread *t = &p->threads[p->thread_id];
4278         struct instruction *ip = t->ip;
4279
4280         /* Structs. */
4281         __instr_regwr_rih_exec(p, t, ip);
4282
4283         /* Thread. */
4284         thread_ip_inc(p);
4285 }
4286
4287 static inline void
4288 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4289 {
4290         struct thread *t = &p->threads[p->thread_id];
4291         struct instruction *ip = t->ip;
4292
4293         /* Structs. */
4294         __instr_regwr_rim_exec(p, t, ip);
4295
4296         /* Thread. */
4297         thread_ip_inc(p);
4298 }
4299
4300 static inline void
4301 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4302 {
4303         struct thread *t = &p->threads[p->thread_id];
4304         struct instruction *ip = t->ip;
4305
4306         /* Structs. */
4307         __instr_regwr_rii_exec(p, t, ip);
4308
4309         /* Thread. */
4310         thread_ip_inc(p);
4311 }
4312
4313 static inline void
4314 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4315 {
4316         struct thread *t = &p->threads[p->thread_id];
4317         struct instruction *ip = t->ip;
4318
4319         /* Structs. */
4320         __instr_regadd_rhh_exec(p, t, ip);
4321
4322         /* Thread. */
4323         thread_ip_inc(p);
4324 }
4325
4326 static inline void
4327 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4328 {
4329         struct thread *t = &p->threads[p->thread_id];
4330         struct instruction *ip = t->ip;
4331
4332         /* Structs. */
4333         __instr_regadd_rhm_exec(p, t, ip);
4334
4335         /* Thread. */
4336         thread_ip_inc(p);
4337 }
4338
4339 static inline void
4340 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4341 {
4342         struct thread *t = &p->threads[p->thread_id];
4343         struct instruction *ip = t->ip;
4344
4345         /* Structs. */
4346         __instr_regadd_rmh_exec(p, t, ip);
4347
4348         /* Thread. */
4349         thread_ip_inc(p);
4350 }
4351
4352 static inline void
4353 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4354 {
4355         struct thread *t = &p->threads[p->thread_id];
4356         struct instruction *ip = t->ip;
4357
4358         /* Structs. */
4359         __instr_regadd_rmm_exec(p, t, ip);
4360
4361         /* Thread. */
4362         thread_ip_inc(p);
4363 }
4364
4365 static inline void
4366 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4367 {
4368         struct thread *t = &p->threads[p->thread_id];
4369         struct instruction *ip = t->ip;
4370
4371         /* Structs. */
4372         __instr_regadd_rhi_exec(p, t, ip);
4373
4374         /* Thread. */
4375         thread_ip_inc(p);
4376 }
4377
4378 static inline void
4379 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4380 {
4381         struct thread *t = &p->threads[p->thread_id];
4382         struct instruction *ip = t->ip;
4383
4384         /* Structs. */
4385         __instr_regadd_rmi_exec(p, t, ip);
4386
4387         /* Thread. */
4388         thread_ip_inc(p);
4389 }
4390
4391 static inline void
4392 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4393 {
4394         struct thread *t = &p->threads[p->thread_id];
4395         struct instruction *ip = t->ip;
4396
4397         /* Structs. */
4398         __instr_regadd_rih_exec(p, t, ip);
4399
4400         /* Thread. */
4401         thread_ip_inc(p);
4402 }
4403
4404 static inline void
4405 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4406 {
4407         struct thread *t = &p->threads[p->thread_id];
4408         struct instruction *ip = t->ip;
4409
4410         /* Structs. */
4411         __instr_regadd_rim_exec(p, t, ip);
4412
4413         /* Thread. */
4414         thread_ip_inc(p);
4415 }
4416
4417 static inline void
4418 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4419 {
4420         struct thread *t = &p->threads[p->thread_id];
4421         struct instruction *ip = t->ip;
4422
4423         /* Structs. */
4424         __instr_regadd_rii_exec(p, t, ip);
4425
4426         /* Thread. */
4427         thread_ip_inc(p);
4428 }
4429
4430 /*
4431  * metarray.
4432  */
4433 static struct metarray *
4434 metarray_find(struct rte_swx_pipeline *p, const char *name);
4435
4436 static int
4437 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4438                             struct action *action,
4439                             char **tokens,
4440                             int n_tokens,
4441                             struct instruction *instr,
4442                             struct instruction_data *data __rte_unused)
4443 {
4444         char *metarray = tokens[1], *idx = tokens[2];
4445         struct metarray *m;
4446         struct field *fidx;
4447         uint32_t idx_struct_id, idx_val;
4448
4449         CHECK(n_tokens == 3, EINVAL);
4450
4451         m = metarray_find(p, metarray);
4452         CHECK(m, EINVAL);
4453
4454         /* METPREFETCH_H, METPREFETCH_M. */
4455         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4456         if (fidx) {
4457                 CHECK(!fidx->var_size, EINVAL);
4458
4459                 instr->type = INSTR_METPREFETCH_M;
4460                 if (idx[0] == 'h')
4461                         instr->type = INSTR_METPREFETCH_H;
4462
4463                 instr->meter.metarray_id = m->id;
4464                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4465                 instr->meter.idx.n_bits = fidx->n_bits;
4466                 instr->meter.idx.offset = fidx->offset / 8;
4467                 return 0;
4468         }
4469
4470         /* METPREFETCH_I. */
4471         idx_val = strtoul(idx, &idx, 0);
4472         CHECK(!idx[0], EINVAL);
4473
4474         instr->type = INSTR_METPREFETCH_I;
4475         instr->meter.metarray_id = m->id;
4476         instr->meter.idx_val = idx_val;
4477         return 0;
4478 }
4479
4480 static int
4481 instr_meter_translate(struct rte_swx_pipeline *p,
4482                       struct action *action,
4483                       char **tokens,
4484                       int n_tokens,
4485                       struct instruction *instr,
4486                       struct instruction_data *data __rte_unused)
4487 {
4488         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4489         char *color_in = tokens[4], *color_out = tokens[5];
4490         struct metarray *m;
4491         struct field *fidx, *flength, *fcin, *fcout;
4492         uint32_t idx_struct_id, length_struct_id;
4493         uint32_t color_in_struct_id, color_out_struct_id;
4494
4495         CHECK(n_tokens == 6, EINVAL);
4496
4497         m = metarray_find(p, metarray);
4498         CHECK(m, EINVAL);
4499
4500         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4501
4502         flength = struct_field_parse(p, action, length, &length_struct_id);
4503         CHECK(flength, EINVAL);
4504         CHECK(!flength->var_size, EINVAL);
4505
4506         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4507
4508         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4509         CHECK(fcout, EINVAL);
4510         CHECK(!fcout->var_size, EINVAL);
4511
4512         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4513         if (fidx && fcin) {
4514                 CHECK(!fidx->var_size, EINVAL);
4515                 CHECK(!fcin->var_size, EINVAL);
4516
4517                 instr->type = INSTR_METER_MMM;
4518                 if (idx[0] == 'h' && length[0] == 'h')
4519                         instr->type = INSTR_METER_HHM;
4520                 if (idx[0] == 'h' && length[0] != 'h')
4521                         instr->type = INSTR_METER_HMM;
4522                 if (idx[0] != 'h' && length[0] == 'h')
4523                         instr->type = INSTR_METER_MHM;
4524
4525                 instr->meter.metarray_id = m->id;
4526
4527                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4528                 instr->meter.idx.n_bits = fidx->n_bits;
4529                 instr->meter.idx.offset = fidx->offset / 8;
4530
4531                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4532                 instr->meter.length.n_bits = flength->n_bits;
4533                 instr->meter.length.offset = flength->offset / 8;
4534
4535                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4536                 instr->meter.color_in.n_bits = fcin->n_bits;
4537                 instr->meter.color_in.offset = fcin->offset / 8;
4538
4539                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4540                 instr->meter.color_out.n_bits = fcout->n_bits;
4541                 instr->meter.color_out.offset = fcout->offset / 8;
4542         }
4543
4544         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4545         if (fidx && !fcin) {
4546                 uint32_t color_in_val;
4547
4548                 CHECK(!fidx->var_size, EINVAL);
4549
4550                 color_in_val = strtoul(color_in, &color_in, 0);
4551                 CHECK(!color_in[0], EINVAL);
4552
4553                 instr->type = INSTR_METER_MMI;
4554                 if (idx[0] == 'h' && length[0] == 'h')
4555                         instr->type = INSTR_METER_HHI;
4556                 if (idx[0] == 'h' && length[0] != 'h')
4557                         instr->type = INSTR_METER_HMI;
4558                 if (idx[0] != 'h' && length[0] == 'h')
4559                         instr->type = INSTR_METER_MHI;
4560
4561                 instr->meter.metarray_id = m->id;
4562
4563                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4564                 instr->meter.idx.n_bits = fidx->n_bits;
4565                 instr->meter.idx.offset = fidx->offset / 8;
4566
4567                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4568                 instr->meter.length.n_bits = flength->n_bits;
4569                 instr->meter.length.offset = flength->offset / 8;
4570
4571                 instr->meter.color_in_val = color_in_val;
4572
4573                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4574                 instr->meter.color_out.n_bits = fcout->n_bits;
4575                 instr->meter.color_out.offset = fcout->offset / 8;
4576         }
4577
4578         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4579         if (!fidx && fcin) {
4580                 uint32_t idx_val;
4581
4582                 idx_val = strtoul(idx, &idx, 0);
4583                 CHECK(!idx[0], EINVAL);
4584
4585                 CHECK(!fcin->var_size, EINVAL);
4586
4587                 instr->type = INSTR_METER_IMM;
4588                 if (length[0] == 'h')
4589                         instr->type = INSTR_METER_IHM;
4590
4591                 instr->meter.metarray_id = m->id;
4592
4593                 instr->meter.idx_val = idx_val;
4594
4595                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4596                 instr->meter.length.n_bits = flength->n_bits;
4597                 instr->meter.length.offset = flength->offset / 8;
4598
4599                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4600                 instr->meter.color_in.n_bits = fcin->n_bits;
4601                 instr->meter.color_in.offset = fcin->offset / 8;
4602
4603                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4604                 instr->meter.color_out.n_bits = fcout->n_bits;
4605                 instr->meter.color_out.offset = fcout->offset / 8;
4606         }
4607
4608         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4609         if (!fidx && !fcin) {
4610                 uint32_t idx_val, color_in_val;
4611
4612                 idx_val = strtoul(idx, &idx, 0);
4613                 CHECK(!idx[0], EINVAL);
4614
4615                 color_in_val = strtoul(color_in, &color_in, 0);
4616                 CHECK(!color_in[0], EINVAL);
4617
4618                 instr->type = INSTR_METER_IMI;
4619                 if (length[0] == 'h')
4620                         instr->type = INSTR_METER_IHI;
4621
4622                 instr->meter.metarray_id = m->id;
4623
4624                 instr->meter.idx_val = idx_val;
4625
4626                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4627                 instr->meter.length.n_bits = flength->n_bits;
4628                 instr->meter.length.offset = flength->offset / 8;
4629
4630                 instr->meter.color_in_val = color_in_val;
4631
4632                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4633                 instr->meter.color_out.n_bits = fcout->n_bits;
4634                 instr->meter.color_out.offset = fcout->offset / 8;
4635         }
4636
4637         return 0;
4638 }
4639
4640 static inline void
4641 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4642 {
4643         struct thread *t = &p->threads[p->thread_id];
4644         struct instruction *ip = t->ip;
4645
4646         /* Structs. */
4647         __instr_metprefetch_h_exec(p, t, ip);
4648
4649         /* Thread. */
4650         thread_ip_inc(p);
4651 }
4652
4653 static inline void
4654 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4655 {
4656         struct thread *t = &p->threads[p->thread_id];
4657         struct instruction *ip = t->ip;
4658
4659         /* Structs. */
4660         __instr_metprefetch_m_exec(p, t, ip);
4661
4662         /* Thread. */
4663         thread_ip_inc(p);
4664 }
4665
4666 static inline void
4667 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4668 {
4669         struct thread *t = &p->threads[p->thread_id];
4670         struct instruction *ip = t->ip;
4671
4672         /* Structs. */
4673         __instr_metprefetch_i_exec(p, t, ip);
4674
4675         /* Thread. */
4676         thread_ip_inc(p);
4677 }
4678
4679 static inline void
4680 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4681 {
4682         struct thread *t = &p->threads[p->thread_id];
4683         struct instruction *ip = t->ip;
4684
4685         /* Structs. */
4686         __instr_meter_hhm_exec(p, t, ip);
4687
4688         /* Thread. */
4689         thread_ip_inc(p);
4690 }
4691
4692 static inline void
4693 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4694 {
4695         struct thread *t = &p->threads[p->thread_id];
4696         struct instruction *ip = t->ip;
4697
4698         /* Structs. */
4699         __instr_meter_hhi_exec(p, t, ip);
4700
4701         /* Thread. */
4702         thread_ip_inc(p);
4703 }
4704
4705 static inline void
4706 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4707 {
4708         struct thread *t = &p->threads[p->thread_id];
4709         struct instruction *ip = t->ip;
4710
4711         /* Structs. */
4712         __instr_meter_hmm_exec(p, t, ip);
4713
4714         /* Thread. */
4715         thread_ip_inc(p);
4716 }
4717
4718 static inline void
4719 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4720 {
4721         struct thread *t = &p->threads[p->thread_id];
4722         struct instruction *ip = t->ip;
4723
4724         /* Structs. */
4725         __instr_meter_hmi_exec(p, t, ip);
4726
4727         /* Thread. */
4728         thread_ip_inc(p);
4729 }
4730
4731 static inline void
4732 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4733 {
4734         struct thread *t = &p->threads[p->thread_id];
4735         struct instruction *ip = t->ip;
4736
4737         /* Structs. */
4738         __instr_meter_mhm_exec(p, t, ip);
4739
4740         /* Thread. */
4741         thread_ip_inc(p);
4742 }
4743
4744 static inline void
4745 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4746 {
4747         struct thread *t = &p->threads[p->thread_id];
4748         struct instruction *ip = t->ip;
4749
4750         /* Structs. */
4751         __instr_meter_mhi_exec(p, t, ip);
4752
4753         /* Thread. */
4754         thread_ip_inc(p);
4755 }
4756
4757 static inline void
4758 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4759 {
4760         struct thread *t = &p->threads[p->thread_id];
4761         struct instruction *ip = t->ip;
4762
4763         /* Structs. */
4764         __instr_meter_mmm_exec(p, t, ip);
4765
4766         /* Thread. */
4767         thread_ip_inc(p);
4768 }
4769
4770 static inline void
4771 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4772 {
4773         struct thread *t = &p->threads[p->thread_id];
4774         struct instruction *ip = t->ip;
4775
4776         /* Structs. */
4777         __instr_meter_mmi_exec(p, t, ip);
4778
4779         /* Thread. */
4780         thread_ip_inc(p);
4781 }
4782
4783 static inline void
4784 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4785 {
4786         struct thread *t = &p->threads[p->thread_id];
4787         struct instruction *ip = t->ip;
4788
4789         /* Structs. */
4790         __instr_meter_ihm_exec(p, t, ip);
4791
4792         /* Thread. */
4793         thread_ip_inc(p);
4794 }
4795
4796 static inline void
4797 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4798 {
4799         struct thread *t = &p->threads[p->thread_id];
4800         struct instruction *ip = t->ip;
4801
4802         /* Structs. */
4803         __instr_meter_ihi_exec(p, t, ip);
4804
4805         /* Thread. */
4806         thread_ip_inc(p);
4807 }
4808
4809 static inline void
4810 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4811 {
4812         struct thread *t = &p->threads[p->thread_id];
4813         struct instruction *ip = t->ip;
4814
4815         /* Structs. */
4816         __instr_meter_imm_exec(p, t, ip);
4817
4818         /* Thread. */
4819         thread_ip_inc(p);
4820 }
4821
4822 static inline void
4823 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4824 {
4825         struct thread *t = &p->threads[p->thread_id];
4826         struct instruction *ip = t->ip;
4827
4828         /* Structs. */
4829         __instr_meter_imi_exec(p, t, ip);
4830
4831         /* Thread. */
4832         thread_ip_inc(p);
4833 }
4834
4835 /*
4836  * jmp.
4837  */
4838 static int
4839 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4840                     struct action *action __rte_unused,
4841                     char **tokens,
4842                     int n_tokens,
4843                     struct instruction *instr,
4844                     struct instruction_data *data)
4845 {
4846         CHECK(n_tokens == 2, EINVAL);
4847
4848         strcpy(data->jmp_label, tokens[1]);
4849
4850         instr->type = INSTR_JMP;
4851         instr->jmp.ip = NULL; /* Resolved later. */
4852         return 0;
4853 }
4854
4855 static int
4856 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4857                           struct action *action __rte_unused,
4858                           char **tokens,
4859                           int n_tokens,
4860                           struct instruction *instr,
4861                           struct instruction_data *data)
4862 {
4863         struct header *h;
4864
4865         CHECK(n_tokens == 3, EINVAL);
4866
4867         strcpy(data->jmp_label, tokens[1]);
4868
4869         h = header_parse(p, tokens[2]);
4870         CHECK(h, EINVAL);
4871
4872         instr->type = INSTR_JMP_VALID;
4873         instr->jmp.ip = NULL; /* Resolved later. */
4874         instr->jmp.header_id = h->id;
4875         return 0;
4876 }
4877
4878 static int
4879 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4880                             struct action *action __rte_unused,
4881                             char **tokens,
4882                             int n_tokens,
4883                             struct instruction *instr,
4884                             struct instruction_data *data)
4885 {
4886         struct header *h;
4887
4888         CHECK(n_tokens == 3, EINVAL);
4889
4890         strcpy(data->jmp_label, tokens[1]);
4891
4892         h = header_parse(p, tokens[2]);
4893         CHECK(h, EINVAL);
4894
4895         instr->type = INSTR_JMP_INVALID;
4896         instr->jmp.ip = NULL; /* Resolved later. */
4897         instr->jmp.header_id = h->id;
4898         return 0;
4899 }
4900
4901 static int
4902 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4903                         struct action *action,
4904                         char **tokens,
4905                         int n_tokens,
4906                         struct instruction *instr,
4907                         struct instruction_data *data)
4908 {
4909         CHECK(!action, EINVAL);
4910         CHECK(n_tokens == 2, EINVAL);
4911
4912         strcpy(data->jmp_label, tokens[1]);
4913
4914         instr->type = INSTR_JMP_HIT;
4915         instr->jmp.ip = NULL; /* Resolved later. */
4916         return 0;
4917 }
4918
4919 static int
4920 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4921                          struct action *action,
4922                          char **tokens,
4923                          int n_tokens,
4924                          struct instruction *instr,
4925                          struct instruction_data *data)
4926 {
4927         CHECK(!action, EINVAL);
4928         CHECK(n_tokens == 2, EINVAL);
4929
4930         strcpy(data->jmp_label, tokens[1]);
4931
4932         instr->type = INSTR_JMP_MISS;
4933         instr->jmp.ip = NULL; /* Resolved later. */
4934         return 0;
4935 }
4936
4937 static int
4938 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4939                                struct action *action,
4940                                char **tokens,
4941                                int n_tokens,
4942                                struct instruction *instr,
4943                                struct instruction_data *data)
4944 {
4945         struct action *a;
4946
4947         CHECK(!action, EINVAL);
4948         CHECK(n_tokens == 3, EINVAL);
4949
4950         strcpy(data->jmp_label, tokens[1]);
4951
4952         a = action_find(p, tokens[2]);
4953         CHECK(a, EINVAL);
4954
4955         instr->type = INSTR_JMP_ACTION_HIT;
4956         instr->jmp.ip = NULL; /* Resolved later. */
4957         instr->jmp.action_id = a->id;
4958         return 0;
4959 }
4960
4961 static int
4962 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4963                                 struct action *action,
4964                                 char **tokens,
4965                                 int n_tokens,
4966                                 struct instruction *instr,
4967                                 struct instruction_data *data)
4968 {
4969         struct action *a;
4970
4971         CHECK(!action, EINVAL);
4972         CHECK(n_tokens == 3, EINVAL);
4973
4974         strcpy(data->jmp_label, tokens[1]);
4975
4976         a = action_find(p, tokens[2]);
4977         CHECK(a, EINVAL);
4978
4979         instr->type = INSTR_JMP_ACTION_MISS;
4980         instr->jmp.ip = NULL; /* Resolved later. */
4981         instr->jmp.action_id = a->id;
4982         return 0;
4983 }
4984
4985 static int
4986 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4987                        struct action *action,
4988                        char **tokens,
4989                        int n_tokens,
4990                        struct instruction *instr,
4991                        struct instruction_data *data)
4992 {
4993         char *a = tokens[2], *b = tokens[3];
4994         struct field *fa, *fb;
4995         uint64_t b_val;
4996         uint32_t a_struct_id, b_struct_id;
4997
4998         CHECK(n_tokens == 4, EINVAL);
4999
5000         strcpy(data->jmp_label, tokens[1]);
5001
5002         fa = struct_field_parse(p, action, a, &a_struct_id);
5003         CHECK(fa, EINVAL);
5004         CHECK(!fa->var_size, EINVAL);
5005
5006         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5007         fb = struct_field_parse(p, action, b, &b_struct_id);
5008         if (fb) {
5009                 CHECK(!fb->var_size, EINVAL);
5010
5011                 instr->type = INSTR_JMP_EQ;
5012                 if (a[0] != 'h' && b[0] == 'h')
5013                         instr->type = INSTR_JMP_EQ_MH;
5014                 if (a[0] == 'h' && b[0] != 'h')
5015                         instr->type = INSTR_JMP_EQ_HM;
5016                 if (a[0] == 'h' && b[0] == 'h')
5017                         instr->type = INSTR_JMP_EQ_HH;
5018                 instr->jmp.ip = NULL; /* Resolved later. */
5019
5020                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5021                 instr->jmp.a.n_bits = fa->n_bits;
5022                 instr->jmp.a.offset = fa->offset / 8;
5023                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5024                 instr->jmp.b.n_bits = fb->n_bits;
5025                 instr->jmp.b.offset = fb->offset / 8;
5026                 return 0;
5027         }
5028
5029         /* JMP_EQ_I. */
5030         b_val = strtoull(b, &b, 0);
5031         CHECK(!b[0], EINVAL);
5032
5033         if (a[0] == 'h')
5034                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5035
5036         instr->type = INSTR_JMP_EQ_I;
5037         instr->jmp.ip = NULL; /* Resolved later. */
5038         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5039         instr->jmp.a.n_bits = fa->n_bits;
5040         instr->jmp.a.offset = fa->offset / 8;
5041         instr->jmp.b_val = b_val;
5042         return 0;
5043 }
5044
5045 static int
5046 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5047                         struct action *action,
5048                         char **tokens,
5049                         int n_tokens,
5050                         struct instruction *instr,
5051                         struct instruction_data *data)
5052 {
5053         char *a = tokens[2], *b = tokens[3];
5054         struct field *fa, *fb;
5055         uint64_t b_val;
5056         uint32_t a_struct_id, b_struct_id;
5057
5058         CHECK(n_tokens == 4, EINVAL);
5059
5060         strcpy(data->jmp_label, tokens[1]);
5061
5062         fa = struct_field_parse(p, action, a, &a_struct_id);
5063         CHECK(fa, EINVAL);
5064         CHECK(!fa->var_size, EINVAL);
5065
5066         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5067         fb = struct_field_parse(p, action, b, &b_struct_id);
5068         if (fb) {
5069                 CHECK(!fb->var_size, EINVAL);
5070
5071                 instr->type = INSTR_JMP_NEQ;
5072                 if (a[0] != 'h' && b[0] == 'h')
5073                         instr->type = INSTR_JMP_NEQ_MH;
5074                 if (a[0] == 'h' && b[0] != 'h')
5075                         instr->type = INSTR_JMP_NEQ_HM;
5076                 if (a[0] == 'h' && b[0] == 'h')
5077                         instr->type = INSTR_JMP_NEQ_HH;
5078                 instr->jmp.ip = NULL; /* Resolved later. */
5079
5080                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5081                 instr->jmp.a.n_bits = fa->n_bits;
5082                 instr->jmp.a.offset = fa->offset / 8;
5083                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5084                 instr->jmp.b.n_bits = fb->n_bits;
5085                 instr->jmp.b.offset = fb->offset / 8;
5086                 return 0;
5087         }
5088
5089         /* JMP_NEQ_I. */
5090         b_val = strtoull(b, &b, 0);
5091         CHECK(!b[0], EINVAL);
5092
5093         if (a[0] == 'h')
5094                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5095
5096         instr->type = INSTR_JMP_NEQ_I;
5097         instr->jmp.ip = NULL; /* Resolved later. */
5098         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5099         instr->jmp.a.n_bits = fa->n_bits;
5100         instr->jmp.a.offset = fa->offset / 8;
5101         instr->jmp.b_val = b_val;
5102         return 0;
5103 }
5104
5105 static int
5106 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5107                        struct action *action,
5108                        char **tokens,
5109                        int n_tokens,
5110                        struct instruction *instr,
5111                        struct instruction_data *data)
5112 {
5113         char *a = tokens[2], *b = tokens[3];
5114         struct field *fa, *fb;
5115         uint64_t b_val;
5116         uint32_t a_struct_id, b_struct_id;
5117
5118         CHECK(n_tokens == 4, EINVAL);
5119
5120         strcpy(data->jmp_label, tokens[1]);
5121
5122         fa = struct_field_parse(p, action, a, &a_struct_id);
5123         CHECK(fa, EINVAL);
5124         CHECK(!fa->var_size, EINVAL);
5125
5126         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5127         fb = struct_field_parse(p, action, b, &b_struct_id);
5128         if (fb) {
5129                 CHECK(!fb->var_size, EINVAL);
5130
5131                 instr->type = INSTR_JMP_LT;
5132                 if (a[0] == 'h' && b[0] != 'h')
5133                         instr->type = INSTR_JMP_LT_HM;
5134                 if (a[0] != 'h' && b[0] == 'h')
5135                         instr->type = INSTR_JMP_LT_MH;
5136                 if (a[0] == 'h' && b[0] == 'h')
5137                         instr->type = INSTR_JMP_LT_HH;
5138                 instr->jmp.ip = NULL; /* Resolved later. */
5139
5140                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5141                 instr->jmp.a.n_bits = fa->n_bits;
5142                 instr->jmp.a.offset = fa->offset / 8;
5143                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5144                 instr->jmp.b.n_bits = fb->n_bits;
5145                 instr->jmp.b.offset = fb->offset / 8;
5146                 return 0;
5147         }
5148
5149         /* JMP_LT_MI, JMP_LT_HI. */
5150         b_val = strtoull(b, &b, 0);
5151         CHECK(!b[0], EINVAL);
5152
5153         instr->type = INSTR_JMP_LT_MI;
5154         if (a[0] == 'h')
5155                 instr->type = INSTR_JMP_LT_HI;
5156         instr->jmp.ip = NULL; /* Resolved later. */
5157
5158         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5159         instr->jmp.a.n_bits = fa->n_bits;
5160         instr->jmp.a.offset = fa->offset / 8;
5161         instr->jmp.b_val = b_val;
5162         return 0;
5163 }
5164
5165 static int
5166 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5167                        struct action *action,
5168                        char **tokens,
5169                        int n_tokens,
5170                        struct instruction *instr,
5171                        struct instruction_data *data)
5172 {
5173         char *a = tokens[2], *b = tokens[3];
5174         struct field *fa, *fb;
5175         uint64_t b_val;
5176         uint32_t a_struct_id, b_struct_id;
5177
5178         CHECK(n_tokens == 4, EINVAL);
5179
5180         strcpy(data->jmp_label, tokens[1]);
5181
5182         fa = struct_field_parse(p, action, a, &a_struct_id);
5183         CHECK(fa, EINVAL);
5184         CHECK(!fa->var_size, EINVAL);
5185
5186         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5187         fb = struct_field_parse(p, action, b, &b_struct_id);
5188         if (fb) {
5189                 CHECK(!fb->var_size, EINVAL);
5190
5191                 instr->type = INSTR_JMP_GT;
5192                 if (a[0] == 'h' && b[0] != 'h')
5193                         instr->type = INSTR_JMP_GT_HM;
5194                 if (a[0] != 'h' && b[0] == 'h')
5195                         instr->type = INSTR_JMP_GT_MH;
5196                 if (a[0] == 'h' && b[0] == 'h')
5197                         instr->type = INSTR_JMP_GT_HH;
5198                 instr->jmp.ip = NULL; /* Resolved later. */
5199
5200                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5201                 instr->jmp.a.n_bits = fa->n_bits;
5202                 instr->jmp.a.offset = fa->offset / 8;
5203                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5204                 instr->jmp.b.n_bits = fb->n_bits;
5205                 instr->jmp.b.offset = fb->offset / 8;
5206                 return 0;
5207         }
5208
5209         /* JMP_GT_MI, JMP_GT_HI. */
5210         b_val = strtoull(b, &b, 0);
5211         CHECK(!b[0], EINVAL);
5212
5213         instr->type = INSTR_JMP_GT_MI;
5214         if (a[0] == 'h')
5215                 instr->type = INSTR_JMP_GT_HI;
5216         instr->jmp.ip = NULL; /* Resolved later. */
5217
5218         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5219         instr->jmp.a.n_bits = fa->n_bits;
5220         instr->jmp.a.offset = fa->offset / 8;
5221         instr->jmp.b_val = b_val;
5222         return 0;
5223 }
5224
5225 static inline void
5226 instr_jmp_exec(struct rte_swx_pipeline *p)
5227 {
5228         struct thread *t = &p->threads[p->thread_id];
5229         struct instruction *ip = t->ip;
5230
5231         TRACE("[Thread %2u] jmp\n", p->thread_id);
5232
5233         thread_ip_set(t, ip->jmp.ip);
5234 }
5235
5236 static inline void
5237 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5238 {
5239         struct thread *t = &p->threads[p->thread_id];
5240         struct instruction *ip = t->ip;
5241         uint32_t header_id = ip->jmp.header_id;
5242
5243         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5244
5245         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5246 }
5247
5248 static inline void
5249 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5250 {
5251         struct thread *t = &p->threads[p->thread_id];
5252         struct instruction *ip = t->ip;
5253         uint32_t header_id = ip->jmp.header_id;
5254
5255         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5256
5257         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5258 }
5259
5260 static inline void
5261 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5262 {
5263         struct thread *t = &p->threads[p->thread_id];
5264         struct instruction *ip = t->ip;
5265         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5266
5267         TRACE("[Thread %2u] jmph\n", p->thread_id);
5268
5269         t->ip = ip_next[t->hit];
5270 }
5271
5272 static inline void
5273 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5274 {
5275         struct thread *t = &p->threads[p->thread_id];
5276         struct instruction *ip = t->ip;
5277         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5278
5279         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5280
5281         t->ip = ip_next[t->hit];
5282 }
5283
5284 static inline void
5285 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5286 {
5287         struct thread *t = &p->threads[p->thread_id];
5288         struct instruction *ip = t->ip;
5289
5290         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5291
5292         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5293 }
5294
5295 static inline void
5296 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5297 {
5298         struct thread *t = &p->threads[p->thread_id];
5299         struct instruction *ip = t->ip;
5300
5301         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5302
5303         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5304 }
5305
5306 static inline void
5307 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5308 {
5309         struct thread *t = &p->threads[p->thread_id];
5310         struct instruction *ip = t->ip;
5311
5312         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5313
5314         JMP_CMP(t, ip, ==);
5315 }
5316
5317 static inline void
5318 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5319 {
5320         struct thread *t = &p->threads[p->thread_id];
5321         struct instruction *ip = t->ip;
5322
5323         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5324
5325         JMP_CMP_MH(t, ip, ==);
5326 }
5327
5328 static inline void
5329 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5330 {
5331         struct thread *t = &p->threads[p->thread_id];
5332         struct instruction *ip = t->ip;
5333
5334         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5335
5336         JMP_CMP_HM(t, ip, ==);
5337 }
5338
5339 static inline void
5340 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5341 {
5342         struct thread *t = &p->threads[p->thread_id];
5343         struct instruction *ip = t->ip;
5344
5345         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5346
5347         JMP_CMP_HH_FAST(t, ip, ==);
5348 }
5349
5350 static inline void
5351 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5352 {
5353         struct thread *t = &p->threads[p->thread_id];
5354         struct instruction *ip = t->ip;
5355
5356         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5357
5358         JMP_CMP_I(t, ip, ==);
5359 }
5360
5361 static inline void
5362 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5363 {
5364         struct thread *t = &p->threads[p->thread_id];
5365         struct instruction *ip = t->ip;
5366
5367         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5368
5369         JMP_CMP(t, ip, !=);
5370 }
5371
5372 static inline void
5373 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5374 {
5375         struct thread *t = &p->threads[p->thread_id];
5376         struct instruction *ip = t->ip;
5377
5378         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5379
5380         JMP_CMP_MH(t, ip, !=);
5381 }
5382
5383 static inline void
5384 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5385 {
5386         struct thread *t = &p->threads[p->thread_id];
5387         struct instruction *ip = t->ip;
5388
5389         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5390
5391         JMP_CMP_HM(t, ip, !=);
5392 }
5393
5394 static inline void
5395 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5396 {
5397         struct thread *t = &p->threads[p->thread_id];
5398         struct instruction *ip = t->ip;
5399
5400         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5401
5402         JMP_CMP_HH_FAST(t, ip, !=);
5403 }
5404
5405 static inline void
5406 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5407 {
5408         struct thread *t = &p->threads[p->thread_id];
5409         struct instruction *ip = t->ip;
5410
5411         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5412
5413         JMP_CMP_I(t, ip, !=);
5414 }
5415
5416 static inline void
5417 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5418 {
5419         struct thread *t = &p->threads[p->thread_id];
5420         struct instruction *ip = t->ip;
5421
5422         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5423
5424         JMP_CMP(t, ip, <);
5425 }
5426
5427 static inline void
5428 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5429 {
5430         struct thread *t = &p->threads[p->thread_id];
5431         struct instruction *ip = t->ip;
5432
5433         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5434
5435         JMP_CMP_MH(t, ip, <);
5436 }
5437
5438 static inline void
5439 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5440 {
5441         struct thread *t = &p->threads[p->thread_id];
5442         struct instruction *ip = t->ip;
5443
5444         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5445
5446         JMP_CMP_HM(t, ip, <);
5447 }
5448
5449 static inline void
5450 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5451 {
5452         struct thread *t = &p->threads[p->thread_id];
5453         struct instruction *ip = t->ip;
5454
5455         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5456
5457         JMP_CMP_HH(t, ip, <);
5458 }
5459
5460 static inline void
5461 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5462 {
5463         struct thread *t = &p->threads[p->thread_id];
5464         struct instruction *ip = t->ip;
5465
5466         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5467
5468         JMP_CMP_MI(t, ip, <);
5469 }
5470
5471 static inline void
5472 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5473 {
5474         struct thread *t = &p->threads[p->thread_id];
5475         struct instruction *ip = t->ip;
5476
5477         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5478
5479         JMP_CMP_HI(t, ip, <);
5480 }
5481
5482 static inline void
5483 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5484 {
5485         struct thread *t = &p->threads[p->thread_id];
5486         struct instruction *ip = t->ip;
5487
5488         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5489
5490         JMP_CMP(t, ip, >);
5491 }
5492
5493 static inline void
5494 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5495 {
5496         struct thread *t = &p->threads[p->thread_id];
5497         struct instruction *ip = t->ip;
5498
5499         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5500
5501         JMP_CMP_MH(t, ip, >);
5502 }
5503
5504 static inline void
5505 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5506 {
5507         struct thread *t = &p->threads[p->thread_id];
5508         struct instruction *ip = t->ip;
5509
5510         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5511
5512         JMP_CMP_HM(t, ip, >);
5513 }
5514
5515 static inline void
5516 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5517 {
5518         struct thread *t = &p->threads[p->thread_id];
5519         struct instruction *ip = t->ip;
5520
5521         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5522
5523         JMP_CMP_HH(t, ip, >);
5524 }
5525
5526 static inline void
5527 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5528 {
5529         struct thread *t = &p->threads[p->thread_id];
5530         struct instruction *ip = t->ip;
5531
5532         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5533
5534         JMP_CMP_MI(t, ip, >);
5535 }
5536
5537 static inline void
5538 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5539 {
5540         struct thread *t = &p->threads[p->thread_id];
5541         struct instruction *ip = t->ip;
5542
5543         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5544
5545         JMP_CMP_HI(t, ip, >);
5546 }
5547
5548 /*
5549  * return.
5550  */
5551 static int
5552 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5553                        struct action *action,
5554                        char **tokens __rte_unused,
5555                        int n_tokens,
5556                        struct instruction *instr,
5557                        struct instruction_data *data __rte_unused)
5558 {
5559         CHECK(action, EINVAL);
5560         CHECK(n_tokens == 1, EINVAL);
5561
5562         instr->type = INSTR_RETURN;
5563         return 0;
5564 }
5565
5566 static inline void
5567 instr_return_exec(struct rte_swx_pipeline *p)
5568 {
5569         struct thread *t = &p->threads[p->thread_id];
5570
5571         TRACE("[Thread %2u] return\n", p->thread_id);
5572
5573         t->ip = t->ret;
5574 }
5575
5576 static int
5577 instr_translate(struct rte_swx_pipeline *p,
5578                 struct action *action,
5579                 char *string,
5580                 struct instruction *instr,
5581                 struct instruction_data *data)
5582 {
5583         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5584         int n_tokens = 0, tpos = 0;
5585
5586         /* Parse the instruction string into tokens. */
5587         for ( ; ; ) {
5588                 char *token;
5589
5590                 token = strtok_r(string, " \t\v", &string);
5591                 if (!token)
5592                         break;
5593
5594                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5595                 CHECK_NAME(token, EINVAL);
5596
5597                 tokens[n_tokens] = token;
5598                 n_tokens++;
5599         }
5600
5601         CHECK(n_tokens, EINVAL);
5602
5603         /* Handle the optional instruction label. */
5604         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5605                 strcpy(data->label, tokens[0]);
5606
5607                 tpos += 2;
5608                 CHECK(n_tokens - tpos, EINVAL);
5609         }
5610
5611         /* Identify the instruction type. */
5612         if (!strcmp(tokens[tpos], "rx"))
5613                 return instr_rx_translate(p,
5614                                           action,
5615                                           &tokens[tpos],
5616                                           n_tokens - tpos,
5617                                           instr,
5618                                           data);
5619
5620         if (!strcmp(tokens[tpos], "tx"))
5621                 return instr_tx_translate(p,
5622                                           action,
5623                                           &tokens[tpos],
5624                                           n_tokens - tpos,
5625                                           instr,
5626                                           data);
5627
5628         if (!strcmp(tokens[tpos], "drop"))
5629                 return instr_drop_translate(p,
5630                                             action,
5631                                             &tokens[tpos],
5632                                             n_tokens - tpos,
5633                                             instr,
5634                                             data);
5635
5636         if (!strcmp(tokens[tpos], "extract"))
5637                 return instr_hdr_extract_translate(p,
5638                                                    action,
5639                                                    &tokens[tpos],
5640                                                    n_tokens - tpos,
5641                                                    instr,
5642                                                    data);
5643
5644         if (!strcmp(tokens[tpos], "lookahead"))
5645                 return instr_hdr_lookahead_translate(p,
5646                                                      action,
5647                                                      &tokens[tpos],
5648                                                      n_tokens - tpos,
5649                                                      instr,
5650                                                      data);
5651
5652         if (!strcmp(tokens[tpos], "emit"))
5653                 return instr_hdr_emit_translate(p,
5654                                                 action,
5655                                                 &tokens[tpos],
5656                                                 n_tokens - tpos,
5657                                                 instr,
5658                                                 data);
5659
5660         if (!strcmp(tokens[tpos], "validate"))
5661                 return instr_hdr_validate_translate(p,
5662                                                     action,
5663                                                     &tokens[tpos],
5664                                                     n_tokens - tpos,
5665                                                     instr,
5666                                                     data);
5667
5668         if (!strcmp(tokens[tpos], "invalidate"))
5669                 return instr_hdr_invalidate_translate(p,
5670                                                       action,
5671                                                       &tokens[tpos],
5672                                                       n_tokens - tpos,
5673                                                       instr,
5674                                                       data);
5675
5676         if (!strcmp(tokens[tpos], "mov"))
5677                 return instr_mov_translate(p,
5678                                            action,
5679                                            &tokens[tpos],
5680                                            n_tokens - tpos,
5681                                            instr,
5682                                            data);
5683
5684         if (!strcmp(tokens[tpos], "add"))
5685                 return instr_alu_add_translate(p,
5686                                                action,
5687                                                &tokens[tpos],
5688                                                n_tokens - tpos,
5689                                                instr,
5690                                                data);
5691
5692         if (!strcmp(tokens[tpos], "sub"))
5693                 return instr_alu_sub_translate(p,
5694                                                action,
5695                                                &tokens[tpos],
5696                                                n_tokens - tpos,
5697                                                instr,
5698                                                data);
5699
5700         if (!strcmp(tokens[tpos], "ckadd"))
5701                 return instr_alu_ckadd_translate(p,
5702                                                  action,
5703                                                  &tokens[tpos],
5704                                                  n_tokens - tpos,
5705                                                  instr,
5706                                                  data);
5707
5708         if (!strcmp(tokens[tpos], "cksub"))
5709                 return instr_alu_cksub_translate(p,
5710                                                  action,
5711                                                  &tokens[tpos],
5712                                                  n_tokens - tpos,
5713                                                  instr,
5714                                                  data);
5715
5716         if (!strcmp(tokens[tpos], "and"))
5717                 return instr_alu_and_translate(p,
5718                                                action,
5719                                                &tokens[tpos],
5720                                                n_tokens - tpos,
5721                                                instr,
5722                                                data);
5723
5724         if (!strcmp(tokens[tpos], "or"))
5725                 return instr_alu_or_translate(p,
5726                                               action,
5727                                               &tokens[tpos],
5728                                               n_tokens - tpos,
5729                                               instr,
5730                                               data);
5731
5732         if (!strcmp(tokens[tpos], "xor"))
5733                 return instr_alu_xor_translate(p,
5734                                                action,
5735                                                &tokens[tpos],
5736                                                n_tokens - tpos,
5737                                                instr,
5738                                                data);
5739
5740         if (!strcmp(tokens[tpos], "shl"))
5741                 return instr_alu_shl_translate(p,
5742                                                action,
5743                                                &tokens[tpos],
5744                                                n_tokens - tpos,
5745                                                instr,
5746                                                data);
5747
5748         if (!strcmp(tokens[tpos], "shr"))
5749                 return instr_alu_shr_translate(p,
5750                                                action,
5751                                                &tokens[tpos],
5752                                                n_tokens - tpos,
5753                                                instr,
5754                                                data);
5755
5756         if (!strcmp(tokens[tpos], "regprefetch"))
5757                 return instr_regprefetch_translate(p,
5758                                                    action,
5759                                                    &tokens[tpos],
5760                                                    n_tokens - tpos,
5761                                                    instr,
5762                                                    data);
5763
5764         if (!strcmp(tokens[tpos], "regrd"))
5765                 return instr_regrd_translate(p,
5766                                              action,
5767                                              &tokens[tpos],
5768                                              n_tokens - tpos,
5769                                              instr,
5770                                              data);
5771
5772         if (!strcmp(tokens[tpos], "regwr"))
5773                 return instr_regwr_translate(p,
5774                                              action,
5775                                              &tokens[tpos],
5776                                              n_tokens - tpos,
5777                                              instr,
5778                                              data);
5779
5780         if (!strcmp(tokens[tpos], "regadd"))
5781                 return instr_regadd_translate(p,
5782                                               action,
5783                                               &tokens[tpos],
5784                                               n_tokens - tpos,
5785                                               instr,
5786                                               data);
5787
5788         if (!strcmp(tokens[tpos], "metprefetch"))
5789                 return instr_metprefetch_translate(p,
5790                                                    action,
5791                                                    &tokens[tpos],
5792                                                    n_tokens - tpos,
5793                                                    instr,
5794                                                    data);
5795
5796         if (!strcmp(tokens[tpos], "meter"))
5797                 return instr_meter_translate(p,
5798                                              action,
5799                                              &tokens[tpos],
5800                                              n_tokens - tpos,
5801                                              instr,
5802                                              data);
5803
5804         if (!strcmp(tokens[tpos], "table"))
5805                 return instr_table_translate(p,
5806                                              action,
5807                                              &tokens[tpos],
5808                                              n_tokens - tpos,
5809                                              instr,
5810                                              data);
5811
5812         if (!strcmp(tokens[tpos], "learn"))
5813                 return instr_learn_translate(p,
5814                                              action,
5815                                              &tokens[tpos],
5816                                              n_tokens - tpos,
5817                                              instr,
5818                                              data);
5819
5820         if (!strcmp(tokens[tpos], "forget"))
5821                 return instr_forget_translate(p,
5822                                               action,
5823                                               &tokens[tpos],
5824                                               n_tokens - tpos,
5825                                               instr,
5826                                               data);
5827
5828         if (!strcmp(tokens[tpos], "extern"))
5829                 return instr_extern_translate(p,
5830                                               action,
5831                                               &tokens[tpos],
5832                                               n_tokens - tpos,
5833                                               instr,
5834                                               data);
5835
5836         if (!strcmp(tokens[tpos], "jmp"))
5837                 return instr_jmp_translate(p,
5838                                            action,
5839                                            &tokens[tpos],
5840                                            n_tokens - tpos,
5841                                            instr,
5842                                            data);
5843
5844         if (!strcmp(tokens[tpos], "jmpv"))
5845                 return instr_jmp_valid_translate(p,
5846                                                  action,
5847                                                  &tokens[tpos],
5848                                                  n_tokens - tpos,
5849                                                  instr,
5850                                                  data);
5851
5852         if (!strcmp(tokens[tpos], "jmpnv"))
5853                 return instr_jmp_invalid_translate(p,
5854                                                    action,
5855                                                    &tokens[tpos],
5856                                                    n_tokens - tpos,
5857                                                    instr,
5858                                                    data);
5859
5860         if (!strcmp(tokens[tpos], "jmph"))
5861                 return instr_jmp_hit_translate(p,
5862                                                action,
5863                                                &tokens[tpos],
5864                                                n_tokens - tpos,
5865                                                instr,
5866                                                data);
5867
5868         if (!strcmp(tokens[tpos], "jmpnh"))
5869                 return instr_jmp_miss_translate(p,
5870                                                 action,
5871                                                 &tokens[tpos],
5872                                                 n_tokens - tpos,
5873                                                 instr,
5874                                                 data);
5875
5876         if (!strcmp(tokens[tpos], "jmpa"))
5877                 return instr_jmp_action_hit_translate(p,
5878                                                       action,
5879                                                       &tokens[tpos],
5880                                                       n_tokens - tpos,
5881                                                       instr,
5882                                                       data);
5883
5884         if (!strcmp(tokens[tpos], "jmpna"))
5885                 return instr_jmp_action_miss_translate(p,
5886                                                        action,
5887                                                        &tokens[tpos],
5888                                                        n_tokens - tpos,
5889                                                        instr,
5890                                                        data);
5891
5892         if (!strcmp(tokens[tpos], "jmpeq"))
5893                 return instr_jmp_eq_translate(p,
5894                                               action,
5895                                               &tokens[tpos],
5896                                               n_tokens - tpos,
5897                                               instr,
5898                                               data);
5899
5900         if (!strcmp(tokens[tpos], "jmpneq"))
5901                 return instr_jmp_neq_translate(p,
5902                                                action,
5903                                                &tokens[tpos],
5904                                                n_tokens - tpos,
5905                                                instr,
5906                                                data);
5907
5908         if (!strcmp(tokens[tpos], "jmplt"))
5909                 return instr_jmp_lt_translate(p,
5910                                               action,
5911                                               &tokens[tpos],
5912                                               n_tokens - tpos,
5913                                               instr,
5914                                               data);
5915
5916         if (!strcmp(tokens[tpos], "jmpgt"))
5917                 return instr_jmp_gt_translate(p,
5918                                               action,
5919                                               &tokens[tpos],
5920                                               n_tokens - tpos,
5921                                               instr,
5922                                               data);
5923
5924         if (!strcmp(tokens[tpos], "return"))
5925                 return instr_return_translate(p,
5926                                               action,
5927                                               &tokens[tpos],
5928                                               n_tokens - tpos,
5929                                               instr,
5930                                               data);
5931
5932         return -EINVAL;
5933 }
5934
5935 static struct instruction_data *
5936 label_find(struct instruction_data *data, uint32_t n, const char *label)
5937 {
5938         uint32_t i;
5939
5940         for (i = 0; i < n; i++)
5941                 if (!strcmp(label, data[i].label))
5942                         return &data[i];
5943
5944         return NULL;
5945 }
5946
5947 static uint32_t
5948 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5949 {
5950         uint32_t count = 0, i;
5951
5952         if (!label[0])
5953                 return 0;
5954
5955         for (i = 0; i < n; i++)
5956                 if (!strcmp(label, data[i].jmp_label))
5957                         count++;
5958
5959         return count;
5960 }
5961
5962 static int
5963 instr_label_check(struct instruction_data *instruction_data,
5964                   uint32_t n_instructions)
5965 {
5966         uint32_t i;
5967
5968         /* Check that all instruction labels are unique. */
5969         for (i = 0; i < n_instructions; i++) {
5970                 struct instruction_data *data = &instruction_data[i];
5971                 char *label = data->label;
5972                 uint32_t j;
5973
5974                 if (!label[0])
5975                         continue;
5976
5977                 for (j = i + 1; j < n_instructions; j++)
5978                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
5979         }
5980
5981         /* Get users for each instruction label. */
5982         for (i = 0; i < n_instructions; i++) {
5983                 struct instruction_data *data = &instruction_data[i];
5984                 char *label = data->label;
5985
5986                 data->n_users = label_is_used(instruction_data,
5987                                               n_instructions,
5988                                               label);
5989         }
5990
5991         return 0;
5992 }
5993
5994 static int
5995 instr_jmp_resolve(struct instruction *instructions,
5996                   struct instruction_data *instruction_data,
5997                   uint32_t n_instructions)
5998 {
5999         uint32_t i;
6000
6001         for (i = 0; i < n_instructions; i++) {
6002                 struct instruction *instr = &instructions[i];
6003                 struct instruction_data *data = &instruction_data[i];
6004                 struct instruction_data *found;
6005
6006                 if (!instruction_is_jmp(instr))
6007                         continue;
6008
6009                 found = label_find(instruction_data,
6010                                    n_instructions,
6011                                    data->jmp_label);
6012                 CHECK(found, EINVAL);
6013
6014                 instr->jmp.ip = &instructions[found - instruction_data];
6015         }
6016
6017         return 0;
6018 }
6019
6020 static int
6021 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6022              struct action *a,
6023              struct instruction *instr,
6024              struct instruction_data *data __rte_unused,
6025              uint32_t n_instructions)
6026 {
6027         if (!a) {
6028                 enum instruction_type type;
6029                 uint32_t i;
6030
6031                 /* Check that the first instruction is rx. */
6032                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6033
6034                 /* Check that there is at least one tx instruction. */
6035                 for (i = 0; i < n_instructions; i++) {
6036                         type = instr[i].type;
6037
6038                         if (instruction_is_tx(type))
6039                                 break;
6040                 }
6041                 CHECK(i < n_instructions, EINVAL);
6042
6043                 /* Check that the last instruction is either tx or unconditional
6044                  * jump.
6045                  */
6046                 type = instr[n_instructions - 1].type;
6047                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6048         }
6049
6050         if (a) {
6051                 enum instruction_type type;
6052                 uint32_t i;
6053
6054                 /* Check that there is at least one return or tx instruction. */
6055                 for (i = 0; i < n_instructions; i++) {
6056                         type = instr[i].type;
6057
6058                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6059                                 break;
6060                 }
6061                 CHECK(i < n_instructions, EINVAL);
6062         }
6063
6064         return 0;
6065 }
6066
6067 static uint32_t
6068 instr_compact(struct instruction *instructions,
6069               struct instruction_data *instruction_data,
6070               uint32_t n_instructions)
6071 {
6072         uint32_t i, pos = 0;
6073
6074         /* Eliminate the invalid instructions that have been optimized out. */
6075         for (i = 0; i < n_instructions; i++) {
6076                 struct instruction *instr = &instructions[i];
6077                 struct instruction_data *data = &instruction_data[i];
6078
6079                 if (data->invalid)
6080                         continue;
6081
6082                 if (i != pos) {
6083                         memcpy(&instructions[pos], instr, sizeof(*instr));
6084                         memcpy(&instruction_data[pos], data, sizeof(*data));
6085                 }
6086
6087                 pos++;
6088         }
6089
6090         return pos;
6091 }
6092
6093 static int
6094 instr_pattern_extract_many_search(struct instruction *instr,
6095                                   struct instruction_data *data,
6096                                   uint32_t n_instr,
6097                                   uint32_t *n_pattern_instr)
6098 {
6099         uint32_t i;
6100
6101         for (i = 0; i < n_instr; i++) {
6102                 if (data[i].invalid)
6103                         break;
6104
6105                 if (instr[i].type != INSTR_HDR_EXTRACT)
6106                         break;
6107
6108                 if (i == RTE_DIM(instr->io.hdr.header_id))
6109                         break;
6110
6111                 if (i && data[i].n_users)
6112                         break;
6113         }
6114
6115         if (i < 2)
6116                 return 0;
6117
6118         *n_pattern_instr = i;
6119         return 1;
6120 }
6121
6122 static void
6123 instr_pattern_extract_many_replace(struct instruction *instr,
6124                                    struct instruction_data *data,
6125                                    uint32_t n_instr)
6126 {
6127         uint32_t i;
6128
6129         for (i = 1; i < n_instr; i++) {
6130                 instr[0].type++;
6131                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6132                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6133                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6134
6135                 data[i].invalid = 1;
6136         }
6137 }
6138
6139 static uint32_t
6140 instr_pattern_extract_many_optimize(struct instruction *instructions,
6141                                     struct instruction_data *instruction_data,
6142                                     uint32_t n_instructions)
6143 {
6144         uint32_t i;
6145
6146         for (i = 0; i < n_instructions; ) {
6147                 struct instruction *instr = &instructions[i];
6148                 struct instruction_data *data = &instruction_data[i];
6149                 uint32_t n_instr = 0;
6150                 int detected;
6151
6152                 /* Extract many. */
6153                 detected = instr_pattern_extract_many_search(instr,
6154                                                              data,
6155                                                              n_instructions - i,
6156                                                              &n_instr);
6157                 if (detected) {
6158                         instr_pattern_extract_many_replace(instr,
6159                                                            data,
6160                                                            n_instr);
6161                         i += n_instr;
6162                         continue;
6163                 }
6164
6165                 /* No pattern starting at the current instruction. */
6166                 i++;
6167         }
6168
6169         /* Eliminate the invalid instructions that have been optimized out. */
6170         n_instructions = instr_compact(instructions,
6171                                        instruction_data,
6172                                        n_instructions);
6173
6174         return n_instructions;
6175 }
6176
6177 static int
6178 instr_pattern_emit_many_tx_search(struct instruction *instr,
6179                                   struct instruction_data *data,
6180                                   uint32_t n_instr,
6181                                   uint32_t *n_pattern_instr)
6182 {
6183         uint32_t i;
6184
6185         for (i = 0; i < n_instr; i++) {
6186                 if (data[i].invalid)
6187                         break;
6188
6189                 if (instr[i].type != INSTR_HDR_EMIT)
6190                         break;
6191
6192                 if (i == RTE_DIM(instr->io.hdr.header_id))
6193                         break;
6194
6195                 if (i && data[i].n_users)
6196                         break;
6197         }
6198
6199         if (!i)
6200                 return 0;
6201
6202         if (!instruction_is_tx(instr[i].type))
6203                 return 0;
6204
6205         if (data[i].n_users)
6206                 return 0;
6207
6208         i++;
6209
6210         *n_pattern_instr = i;
6211         return 1;
6212 }
6213
6214 static void
6215 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6216                                    struct instruction_data *data,
6217                                    uint32_t n_instr)
6218 {
6219         uint32_t i;
6220
6221         /* Any emit instruction in addition to the first one. */
6222         for (i = 1; i < n_instr - 1; i++) {
6223                 instr[0].type++;
6224                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6225                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6226                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6227
6228                 data[i].invalid = 1;
6229         }
6230
6231         /* The TX instruction is the last one in the pattern. */
6232         instr[0].type++;
6233         instr[0].io.io.offset = instr[i].io.io.offset;
6234         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6235         data[i].invalid = 1;
6236 }
6237
6238 static uint32_t
6239 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6240                                     struct instruction_data *instruction_data,
6241                                     uint32_t n_instructions)
6242 {
6243         uint32_t i;
6244
6245         for (i = 0; i < n_instructions; ) {
6246                 struct instruction *instr = &instructions[i];
6247                 struct instruction_data *data = &instruction_data[i];
6248                 uint32_t n_instr = 0;
6249                 int detected;
6250
6251                 /* Emit many + TX. */
6252                 detected = instr_pattern_emit_many_tx_search(instr,
6253                                                              data,
6254                                                              n_instructions - i,
6255                                                              &n_instr);
6256                 if (detected) {
6257                         instr_pattern_emit_many_tx_replace(instr,
6258                                                            data,
6259                                                            n_instr);
6260                         i += n_instr;
6261                         continue;
6262                 }
6263
6264                 /* No pattern starting at the current instruction. */
6265                 i++;
6266         }
6267
6268         /* Eliminate the invalid instructions that have been optimized out. */
6269         n_instructions = instr_compact(instructions,
6270                                        instruction_data,
6271                                        n_instructions);
6272
6273         return n_instructions;
6274 }
6275
6276 static uint32_t
6277 action_arg_src_mov_count(struct action *a,
6278                          uint32_t arg_id,
6279                          struct instruction *instructions,
6280                          struct instruction_data *instruction_data,
6281                          uint32_t n_instructions);
6282
6283 static int
6284 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6285                                       struct action *a,
6286                                       struct instruction *instr,
6287                                       struct instruction_data *data,
6288                                       uint32_t n_instr,
6289                                       struct instruction *instructions,
6290                                       struct instruction_data *instruction_data,
6291                                       uint32_t n_instructions,
6292                                       uint32_t *n_pattern_instr)
6293 {
6294         struct header *h;
6295         uint32_t src_field_id, i, j;
6296
6297         /* Prerequisites. */
6298         if (!a || !a->st)
6299                 return 0;
6300
6301         /* First instruction: MOV_HM. */
6302         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6303                 return 0;
6304
6305         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6306         if (!h || h->st->var_size)
6307                 return 0;
6308
6309         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6310                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6311                         break;
6312
6313         if (src_field_id == a->st->n_fields)
6314                 return 0;
6315
6316         if (instr[0].mov.dst.offset ||
6317             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6318             instr[0].mov.src.struct_id ||
6319             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6320             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6321                 return 0;
6322
6323         if ((n_instr < h->st->n_fields + 1) ||
6324              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6325                 return 0;
6326
6327         /* Subsequent instructions: MOV_HM. */
6328         for (i = 1; i < h->st->n_fields; i++)
6329                 if (data[i].invalid ||
6330                     data[i].n_users ||
6331                     (instr[i].type != INSTR_MOV_HM) ||
6332                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6333                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6334                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6335                     instr[i].mov.src.struct_id ||
6336                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6337                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6338                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6339                         return 0;
6340
6341         /* Last instruction: HDR_VALIDATE. */
6342         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6343             (instr[i].valid.header_id != h->id))
6344                 return 0;
6345
6346         /* Check that none of the action args that are used as source for this
6347          * DMA transfer are not used as source in any other mov instruction.
6348          */
6349         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6350                 uint32_t n_users;
6351
6352                 n_users = action_arg_src_mov_count(a,
6353                                                    j,
6354                                                    instructions,
6355                                                    instruction_data,
6356                                                    n_instructions);
6357                 if (n_users > 1)
6358                         return 0;
6359         }
6360
6361         *n_pattern_instr = 1 + i;
6362         return 1;
6363 }
6364
6365 static void
6366 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6367                                        struct action *a,
6368                                        struct instruction *instr,
6369                                        struct instruction_data *data,
6370                                        uint32_t n_instr)
6371 {
6372         struct header *h;
6373         uint32_t src_field_id, src_offset, i;
6374
6375         /* Read from the instructions before they are modified. */
6376         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6377         if (!h)
6378                 return;
6379
6380         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6381                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6382                         break;
6383
6384         if (src_field_id == a->st->n_fields)
6385                 return;
6386
6387         src_offset = instr[0].mov.src.offset;
6388
6389         /* Modify the instructions. */
6390         instr[0].type = INSTR_DMA_HT;
6391         instr[0].dma.dst.header_id[0] = h->id;
6392         instr[0].dma.dst.struct_id[0] = h->struct_id;
6393         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6394         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6395
6396         for (i = 1; i < n_instr; i++)
6397                 data[i].invalid = 1;
6398
6399         /* Update the endianness of the action arguments to header endianness. */
6400         for (i = 0; i < h->st->n_fields; i++)
6401                 a->args_endianness[src_field_id + i] = 1;
6402 }
6403
6404 static uint32_t
6405 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6406                                         struct action *a,
6407                                         struct instruction *instructions,
6408                                         struct instruction_data *instruction_data,
6409                                         uint32_t n_instructions)
6410 {
6411         uint32_t i;
6412
6413         if (!a || !a->st)
6414                 return n_instructions;
6415
6416         for (i = 0; i < n_instructions; ) {
6417                 struct instruction *instr = &instructions[i];
6418                 struct instruction_data *data = &instruction_data[i];
6419                 uint32_t n_instr = 0;
6420                 int detected;
6421
6422                 /* Mov all + validate. */
6423                 detected = instr_pattern_mov_all_validate_search(p,
6424                                                                  a,
6425                                                                  instr,
6426                                                                  data,
6427                                                                  n_instructions - i,
6428                                                                  instructions,
6429                                                                  instruction_data,
6430                                                                  n_instructions,
6431                                                                  &n_instr);
6432                 if (detected) {
6433                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6434                         i += n_instr;
6435                         continue;
6436                 }
6437
6438                 /* No pattern starting at the current instruction. */
6439                 i++;
6440         }
6441
6442         /* Eliminate the invalid instructions that have been optimized out. */
6443         n_instructions = instr_compact(instructions,
6444                                        instruction_data,
6445                                        n_instructions);
6446
6447         return n_instructions;
6448 }
6449
6450 static int
6451 instr_pattern_dma_many_search(struct instruction *instr,
6452                               struct instruction_data *data,
6453                               uint32_t n_instr,
6454                               uint32_t *n_pattern_instr)
6455 {
6456         uint32_t i;
6457
6458         for (i = 0; i < n_instr; i++) {
6459                 if (data[i].invalid)
6460                         break;
6461
6462                 if (instr[i].type != INSTR_DMA_HT)
6463                         break;
6464
6465                 if (i == RTE_DIM(instr->dma.dst.header_id))
6466                         break;
6467
6468                 if (i && data[i].n_users)
6469                         break;
6470         }
6471
6472         if (i < 2)
6473                 return 0;
6474
6475         *n_pattern_instr = i;
6476         return 1;
6477 }
6478
6479 static void
6480 instr_pattern_dma_many_replace(struct instruction *instr,
6481                                struct instruction_data *data,
6482                                uint32_t n_instr)
6483 {
6484         uint32_t i;
6485
6486         for (i = 1; i < n_instr; i++) {
6487                 instr[0].type++;
6488                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6489                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6490                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6491                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6492
6493                 data[i].invalid = 1;
6494         }
6495 }
6496
6497 static uint32_t
6498 instr_pattern_dma_many_optimize(struct instruction *instructions,
6499                struct instruction_data *instruction_data,
6500                uint32_t n_instructions)
6501 {
6502         uint32_t i;
6503
6504         for (i = 0; i < n_instructions; ) {
6505                 struct instruction *instr = &instructions[i];
6506                 struct instruction_data *data = &instruction_data[i];
6507                 uint32_t n_instr = 0;
6508                 int detected;
6509
6510                 /* DMA many. */
6511                 detected = instr_pattern_dma_many_search(instr,
6512                                                          data,
6513                                                          n_instructions - i,
6514                                                          &n_instr);
6515                 if (detected) {
6516                         instr_pattern_dma_many_replace(instr, data, n_instr);
6517                         i += n_instr;
6518                         continue;
6519                 }
6520
6521                 /* No pattern starting at the current instruction. */
6522                 i++;
6523         }
6524
6525         /* Eliminate the invalid instructions that have been optimized out. */
6526         n_instructions = instr_compact(instructions,
6527                                        instruction_data,
6528                                        n_instructions);
6529
6530         return n_instructions;
6531 }
6532
6533 static uint32_t
6534 instr_optimize(struct rte_swx_pipeline *p,
6535                struct action *a,
6536                struct instruction *instructions,
6537                struct instruction_data *instruction_data,
6538                uint32_t n_instructions)
6539 {
6540         /* Extract many. */
6541         n_instructions = instr_pattern_extract_many_optimize(instructions,
6542                                                              instruction_data,
6543                                                              n_instructions);
6544
6545         /* Emit many + TX. */
6546         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6547                                                              instruction_data,
6548                                                              n_instructions);
6549
6550         /* Mov all + validate. */
6551         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6552                                                                  a,
6553                                                                  instructions,
6554                                                                  instruction_data,
6555                                                                  n_instructions);
6556
6557         /* DMA many. */
6558         n_instructions = instr_pattern_dma_many_optimize(instructions,
6559                                                          instruction_data,
6560                                                          n_instructions);
6561
6562         return n_instructions;
6563 }
6564
6565 static int
6566 instruction_config(struct rte_swx_pipeline *p,
6567                    struct action *a,
6568                    const char **instructions,
6569                    uint32_t n_instructions)
6570 {
6571         struct instruction *instr = NULL;
6572         struct instruction_data *data = NULL;
6573         int err = 0;
6574         uint32_t i;
6575
6576         CHECK(n_instructions, EINVAL);
6577         CHECK(instructions, EINVAL);
6578         for (i = 0; i < n_instructions; i++)
6579                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6580
6581         /* Memory allocation. */
6582         instr = calloc(n_instructions, sizeof(struct instruction));
6583         if (!instr) {
6584                 err = -ENOMEM;
6585                 goto error;
6586         }
6587
6588         data = calloc(n_instructions, sizeof(struct instruction_data));
6589         if (!data) {
6590                 err = -ENOMEM;
6591                 goto error;
6592         }
6593
6594         for (i = 0; i < n_instructions; i++) {
6595                 char *string = strdup(instructions[i]);
6596                 if (!string) {
6597                         err = -ENOMEM;
6598                         goto error;
6599                 }
6600
6601                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6602                 if (err) {
6603                         free(string);
6604                         goto error;
6605                 }
6606
6607                 free(string);
6608         }
6609
6610         err = instr_label_check(data, n_instructions);
6611         if (err)
6612                 goto error;
6613
6614         err = instr_verify(p, a, instr, data, n_instructions);
6615         if (err)
6616                 goto error;
6617
6618         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6619
6620         err = instr_jmp_resolve(instr, data, n_instructions);
6621         if (err)
6622                 goto error;
6623
6624         if (a) {
6625                 a->instructions = instr;
6626                 a->instruction_data = data;
6627                 a->n_instructions = n_instructions;
6628         } else {
6629                 p->instructions = instr;
6630                 p->instruction_data = data;
6631                 p->n_instructions = n_instructions;
6632         }
6633
6634         return 0;
6635
6636 error:
6637         free(data);
6638         free(instr);
6639         return err;
6640 }
6641
6642 static instr_exec_t instruction_table[] = {
6643         [INSTR_RX] = instr_rx_exec,
6644         [INSTR_TX] = instr_tx_exec,
6645         [INSTR_TX_I] = instr_tx_i_exec,
6646
6647         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6648         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6649         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6650         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6651         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6652         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6653         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6654         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6655         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6656         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6657
6658         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6659         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6660         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6661         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6662         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6663         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6664         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6665         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6666         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6667
6668         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6669         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6670
6671         [INSTR_MOV] = instr_mov_exec,
6672         [INSTR_MOV_MH] = instr_mov_mh_exec,
6673         [INSTR_MOV_HM] = instr_mov_hm_exec,
6674         [INSTR_MOV_HH] = instr_mov_hh_exec,
6675         [INSTR_MOV_I] = instr_mov_i_exec,
6676
6677         [INSTR_DMA_HT] = instr_dma_ht_exec,
6678         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6679         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6680         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6681         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6682         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6683         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6684         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6685
6686         [INSTR_ALU_ADD] = instr_alu_add_exec,
6687         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6688         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6689         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6690         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6691         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6692
6693         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6694         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6695         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6696         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6697         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6698         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6699
6700         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6701         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6702         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6703         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6704
6705         [INSTR_ALU_AND] = instr_alu_and_exec,
6706         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6707         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6708         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6709         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6710
6711         [INSTR_ALU_OR] = instr_alu_or_exec,
6712         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6713         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6714         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6715         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6716
6717         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6718         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6719         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6720         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6721         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6722
6723         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6724         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6725         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6726         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6727         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6728         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6729
6730         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6731         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6732         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6733         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6734         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6735         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6736
6737         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6738         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6739         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6740
6741         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6742         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6743         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6744         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6745         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6746         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6747
6748         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6749         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6750         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6751         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6752         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6753         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6754         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6755         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6756         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6757
6758         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6759         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6760         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6761         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6762         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6763         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6764         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6765         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6766         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6767
6768         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6769         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6770         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6771
6772         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6773         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6774         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6775         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6776         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6777         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6778         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6779         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6780         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6781         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6782         [INSTR_METER_IMM] = instr_meter_imm_exec,
6783         [INSTR_METER_IMI] = instr_meter_imi_exec,
6784
6785         [INSTR_TABLE] = instr_table_exec,
6786         [INSTR_TABLE_AF] = instr_table_af_exec,
6787         [INSTR_SELECTOR] = instr_selector_exec,
6788         [INSTR_LEARNER] = instr_learner_exec,
6789         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6790         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6791         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6792         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6793         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6794
6795         [INSTR_JMP] = instr_jmp_exec,
6796         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6797         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6798         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6799         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6800         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6801         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6802
6803         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6804         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6805         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6806         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6807         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6808
6809         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6810         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6811         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6812         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6813         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6814
6815         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6816         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6817         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6818         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6819         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6820         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6821
6822         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6823         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6824         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6825         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6826         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6827         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6828
6829         [INSTR_RETURN] = instr_return_exec,
6830 };
6831
6832 static int
6833 instruction_table_build(struct rte_swx_pipeline *p)
6834 {
6835         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6836                                       sizeof(struct instr_exec_t *));
6837         if (!p->instruction_table)
6838                 return -EINVAL;
6839
6840         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6841
6842         return 0;
6843 }
6844
6845 static void
6846 instruction_table_build_free(struct rte_swx_pipeline *p)
6847 {
6848         if (!p->instruction_table)
6849                 return;
6850
6851         free(p->instruction_table);
6852         p->instruction_table = NULL;
6853 }
6854
6855 static void
6856 instruction_table_free(struct rte_swx_pipeline *p)
6857 {
6858         instruction_table_build_free(p);
6859 }
6860
6861 static inline void
6862 instr_exec(struct rte_swx_pipeline *p)
6863 {
6864         struct thread *t = &p->threads[p->thread_id];
6865         struct instruction *ip = t->ip;
6866         instr_exec_t instr = p->instruction_table[ip->type];
6867
6868         instr(p);
6869 }
6870
6871 /*
6872  * Action.
6873  */
6874 static struct action *
6875 action_find(struct rte_swx_pipeline *p, const char *name)
6876 {
6877         struct action *elem;
6878
6879         if (!name)
6880                 return NULL;
6881
6882         TAILQ_FOREACH(elem, &p->actions, node)
6883                 if (strcmp(elem->name, name) == 0)
6884                         return elem;
6885
6886         return NULL;
6887 }
6888
6889 static struct action *
6890 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6891 {
6892         struct action *action = NULL;
6893
6894         TAILQ_FOREACH(action, &p->actions, node)
6895                 if (action->id == id)
6896                         return action;
6897
6898         return NULL;
6899 }
6900
6901 static struct field *
6902 action_field_find(struct action *a, const char *name)
6903 {
6904         return a->st ? struct_type_field_find(a->st, name) : NULL;
6905 }
6906
6907 static struct field *
6908 action_field_parse(struct action *action, const char *name)
6909 {
6910         if (name[0] != 't' || name[1] != '.')
6911                 return NULL;
6912
6913         return action_field_find(action, &name[2]);
6914 }
6915
6916 static int
6917 action_has_nbo_args(struct action *a)
6918 {
6919         uint32_t i;
6920
6921         /* Return if the action does not have any args. */
6922         if (!a->st)
6923                 return 0; /* FALSE */
6924
6925         for (i = 0; i < a->st->n_fields; i++)
6926                 if (a->args_endianness[i])
6927                         return 1; /* TRUE */
6928
6929         return 0; /* FALSE */
6930 }
6931
6932 static int
6933 action_does_learning(struct action *a)
6934 {
6935         uint32_t i;
6936
6937         for (i = 0; i < a->n_instructions; i++)
6938                 switch (a->instructions[i].type) {
6939                 case INSTR_LEARNER_LEARN:
6940                         return 1; /* TRUE */
6941
6942                 case INSTR_LEARNER_FORGET:
6943                         return 1; /* TRUE */
6944
6945                 default:
6946                         continue;
6947                 }
6948
6949         return 0; /* FALSE */
6950 }
6951
6952 int
6953 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6954                                const char *name,
6955                                const char *args_struct_type_name,
6956                                const char **instructions,
6957                                uint32_t n_instructions)
6958 {
6959         struct struct_type *args_struct_type = NULL;
6960         struct action *a;
6961         int err;
6962
6963         CHECK(p, EINVAL);
6964
6965         CHECK_NAME(name, EINVAL);
6966         CHECK(!action_find(p, name), EEXIST);
6967
6968         if (args_struct_type_name) {
6969                 CHECK_NAME(args_struct_type_name, EINVAL);
6970                 args_struct_type = struct_type_find(p, args_struct_type_name);
6971                 CHECK(args_struct_type, EINVAL);
6972                 CHECK(!args_struct_type->var_size, EINVAL);
6973         }
6974
6975         /* Node allocation. */
6976         a = calloc(1, sizeof(struct action));
6977         CHECK(a, ENOMEM);
6978         if (args_struct_type) {
6979                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6980                 if (!a->args_endianness) {
6981                         free(a);
6982                         CHECK(0, ENOMEM);
6983                 }
6984         }
6985
6986         /* Node initialization. */
6987         strcpy(a->name, name);
6988         a->st = args_struct_type;
6989         a->id = p->n_actions;
6990
6991         /* Instruction translation. */
6992         err = instruction_config(p, a, instructions, n_instructions);
6993         if (err) {
6994                 free(a->args_endianness);
6995                 free(a);
6996                 return err;
6997         }
6998
6999         /* Node add to tailq. */
7000         TAILQ_INSERT_TAIL(&p->actions, a, node);
7001         p->n_actions++;
7002
7003         return 0;
7004 }
7005
7006 static int
7007 action_build(struct rte_swx_pipeline *p)
7008 {
7009         struct action *action;
7010
7011         /* p->action_instructions. */
7012         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7013         CHECK(p->action_instructions, ENOMEM);
7014
7015         TAILQ_FOREACH(action, &p->actions, node)
7016                 p->action_instructions[action->id] = action->instructions;
7017
7018         /* p->action_funcs. */
7019         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7020         CHECK(p->action_funcs, ENOMEM);
7021
7022         return 0;
7023 }
7024
7025 static void
7026 action_build_free(struct rte_swx_pipeline *p)
7027 {
7028         free(p->action_funcs);
7029         p->action_funcs = NULL;
7030
7031         free(p->action_instructions);
7032         p->action_instructions = NULL;
7033 }
7034
7035 static void
7036 action_free(struct rte_swx_pipeline *p)
7037 {
7038         action_build_free(p);
7039
7040         for ( ; ; ) {
7041                 struct action *action;
7042
7043                 action = TAILQ_FIRST(&p->actions);
7044                 if (!action)
7045                         break;
7046
7047                 TAILQ_REMOVE(&p->actions, action, node);
7048                 free(action->instruction_data);
7049                 free(action->instructions);
7050                 free(action);
7051         }
7052 }
7053
7054 static uint32_t
7055 action_arg_src_mov_count(struct action *a,
7056                          uint32_t arg_id,
7057                          struct instruction *instructions,
7058                          struct instruction_data *instruction_data,
7059                          uint32_t n_instructions)
7060 {
7061         uint32_t offset, n_users = 0, i;
7062
7063         if (!a->st ||
7064             (arg_id >= a->st->n_fields) ||
7065             !instructions ||
7066             !instruction_data ||
7067             !n_instructions)
7068                 return 0;
7069
7070         offset = a->st->fields[arg_id].offset / 8;
7071
7072         for (i = 0; i < n_instructions; i++) {
7073                 struct instruction *instr = &instructions[i];
7074                 struct instruction_data *data = &instruction_data[i];
7075
7076                 if (data->invalid ||
7077                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7078                     instr->mov.src.struct_id ||
7079                     (instr->mov.src.offset != offset))
7080                         continue;
7081
7082                 n_users++;
7083         }
7084
7085         return n_users;
7086 }
7087
7088 /*
7089  * Table.
7090  */
7091 static struct table_type *
7092 table_type_find(struct rte_swx_pipeline *p, const char *name)
7093 {
7094         struct table_type *elem;
7095
7096         TAILQ_FOREACH(elem, &p->table_types, node)
7097                 if (strcmp(elem->name, name) == 0)
7098                         return elem;
7099
7100         return NULL;
7101 }
7102
7103 static struct table_type *
7104 table_type_resolve(struct rte_swx_pipeline *p,
7105                    const char *recommended_type_name,
7106                    enum rte_swx_table_match_type match_type)
7107 {
7108         struct table_type *elem;
7109
7110         /* Only consider the recommended type if the match type is correct. */
7111         if (recommended_type_name)
7112                 TAILQ_FOREACH(elem, &p->table_types, node)
7113                         if (!strcmp(elem->name, recommended_type_name) &&
7114                             (elem->match_type == match_type))
7115                                 return elem;
7116
7117         /* Ignore the recommended type and get the first element with this match
7118          * type.
7119          */
7120         TAILQ_FOREACH(elem, &p->table_types, node)
7121                 if (elem->match_type == match_type)
7122                         return elem;
7123
7124         return NULL;
7125 }
7126
7127 static struct table *
7128 table_find(struct rte_swx_pipeline *p, const char *name)
7129 {
7130         struct table *elem;
7131
7132         TAILQ_FOREACH(elem, &p->tables, node)
7133                 if (strcmp(elem->name, name) == 0)
7134                         return elem;
7135
7136         return NULL;
7137 }
7138
7139 static struct table *
7140 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7141 {
7142         struct table *table = NULL;
7143
7144         TAILQ_FOREACH(table, &p->tables, node)
7145                 if (table->id == id)
7146                         return table;
7147
7148         return NULL;
7149 }
7150
7151 int
7152 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7153                                      const char *name,
7154                                      enum rte_swx_table_match_type match_type,
7155                                      struct rte_swx_table_ops *ops)
7156 {
7157         struct table_type *elem;
7158
7159         CHECK(p, EINVAL);
7160
7161         CHECK_NAME(name, EINVAL);
7162         CHECK(!table_type_find(p, name), EEXIST);
7163
7164         CHECK(ops, EINVAL);
7165         CHECK(ops->create, EINVAL);
7166         CHECK(ops->lkp, EINVAL);
7167         CHECK(ops->free, EINVAL);
7168
7169         /* Node allocation. */
7170         elem = calloc(1, sizeof(struct table_type));
7171         CHECK(elem, ENOMEM);
7172
7173         /* Node initialization. */
7174         strcpy(elem->name, name);
7175         elem->match_type = match_type;
7176         memcpy(&elem->ops, ops, sizeof(*ops));
7177
7178         /* Node add to tailq. */
7179         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7180
7181         return 0;
7182 }
7183
7184 static int
7185 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7186                          uint32_t n_fields,
7187                          enum rte_swx_table_match_type *match_type)
7188 {
7189         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7190
7191         for (i = 0; i < n_fields; i++) {
7192                 struct rte_swx_match_field_params  *f = &fields[i];
7193
7194                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7195                         n_fields_em++;
7196
7197                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7198                         n_fields_lpm++;
7199         }
7200
7201         if ((n_fields_lpm > 1) ||
7202             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7203                 return -EINVAL;
7204
7205         *match_type = (n_fields_em == n_fields) ?
7206                        RTE_SWX_TABLE_MATCH_EXACT :
7207                        RTE_SWX_TABLE_MATCH_WILDCARD;
7208
7209         return 0;
7210 }
7211
7212 static int
7213 table_match_fields_check(struct rte_swx_pipeline *p,
7214                          struct rte_swx_pipeline_table_params *params,
7215                          struct header **header)
7216 {
7217         struct header *h0 = NULL;
7218         struct field *hf, *mf;
7219         uint32_t *offset = NULL, i;
7220         int status = 0;
7221
7222         /* Return if no match fields. */
7223         if (!params->n_fields) {
7224                 if (params->fields) {
7225                         status = -EINVAL;
7226                         goto end;
7227                 }
7228
7229                 if (header)
7230                         *header = NULL;
7231
7232                 return 0;
7233         }
7234
7235         /* Memory allocation. */
7236         offset = calloc(params->n_fields, sizeof(uint32_t));
7237         if (!offset) {
7238                 status = -ENOMEM;
7239                 goto end;
7240         }
7241
7242         /* Check that all the match fields belong to either the same header or
7243          * to the meta-data.
7244          */
7245         hf = header_field_parse(p, params->fields[0].name, &h0);
7246         mf = metadata_field_parse(p, params->fields[0].name);
7247         if ((!hf && !mf) || (hf && hf->var_size)) {
7248                 status = -EINVAL;
7249                 goto end;
7250         }
7251
7252         offset[0] = h0 ? hf->offset : mf->offset;
7253
7254         for (i = 1; i < params->n_fields; i++)
7255                 if (h0) {
7256                         struct header *h;
7257
7258                         hf = header_field_parse(p, params->fields[i].name, &h);
7259                         if (!hf || (h->id != h0->id) || hf->var_size) {
7260                                 status = -EINVAL;
7261                                 goto end;
7262                         }
7263
7264                         offset[i] = hf->offset;
7265                 } else {
7266                         mf = metadata_field_parse(p, params->fields[i].name);
7267                         if (!mf) {
7268                                 status = -EINVAL;
7269                                 goto end;
7270                         }
7271
7272                         offset[i] = mf->offset;
7273                 }
7274
7275         /* Check that there are no duplicated match fields. */
7276         for (i = 0; i < params->n_fields; i++) {
7277                 uint32_t j;
7278
7279                 for (j = 0; j < i; j++)
7280                         if (offset[j] == offset[i]) {
7281                                 status = -EINVAL;
7282                                 goto end;
7283                         }
7284         }
7285
7286         /* Return. */
7287         if (header)
7288                 *header = h0;
7289
7290 end:
7291         free(offset);
7292         return status;
7293 }
7294
7295 int
7296 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7297                               const char *name,
7298                               struct rte_swx_pipeline_table_params *params,
7299                               const char *recommended_table_type_name,
7300                               const char *args,
7301                               uint32_t size)
7302 {
7303         struct table_type *type;
7304         struct table *t = NULL;
7305         struct action *default_action;
7306         struct header *header = NULL;
7307         uint32_t action_data_size_max = 0, i;
7308         int status = 0;
7309
7310         CHECK(p, EINVAL);
7311
7312         CHECK_NAME(name, EINVAL);
7313         CHECK(!table_find(p, name), EEXIST);
7314         CHECK(!selector_find(p, name), EEXIST);
7315         CHECK(!learner_find(p, name), EEXIST);
7316
7317         CHECK(params, EINVAL);
7318
7319         /* Match checks. */
7320         status = table_match_fields_check(p, params, &header);
7321         if (status)
7322                 return status;
7323
7324         /* Action checks. */
7325         CHECK(params->n_actions, EINVAL);
7326         CHECK(params->action_names, EINVAL);
7327         for (i = 0; i < params->n_actions; i++) {
7328                 const char *action_name = params->action_names[i];
7329                 struct action *a;
7330                 uint32_t action_data_size;
7331                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7332
7333                 CHECK_NAME(action_name, EINVAL);
7334
7335                 a = action_find(p, action_name);
7336                 CHECK(a, EINVAL);
7337                 CHECK(!action_does_learning(a), EINVAL);
7338
7339                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7340                 if (action_data_size > action_data_size_max)
7341                         action_data_size_max = action_data_size;
7342
7343                 if (params->action_is_for_table_entries)
7344                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7345                 if (params->action_is_for_default_entry)
7346                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7347                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7348         }
7349
7350         CHECK_NAME(params->default_action_name, EINVAL);
7351         for (i = 0; i < p->n_actions; i++)
7352                 if (!strcmp(params->action_names[i],
7353                             params->default_action_name))
7354                         break;
7355         CHECK(i < params->n_actions, EINVAL);
7356         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7357               EINVAL);
7358
7359         default_action = action_find(p, params->default_action_name);
7360         CHECK((default_action->st && params->default_action_data) ||
7361               !params->default_action_data, EINVAL);
7362
7363         /* Table type checks. */
7364         if (recommended_table_type_name)
7365                 CHECK_NAME(recommended_table_type_name, EINVAL);
7366
7367         if (params->n_fields) {
7368                 enum rte_swx_table_match_type match_type;
7369
7370                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7371                 if (status)
7372                         return status;
7373
7374                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7375                 CHECK(type, EINVAL);
7376         } else {
7377                 type = NULL;
7378         }
7379
7380         /* Memory allocation. */
7381         t = calloc(1, sizeof(struct table));
7382         if (!t)
7383                 goto nomem;
7384
7385         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7386         if (!t->fields)
7387                 goto nomem;
7388
7389         t->actions = calloc(params->n_actions, sizeof(struct action *));
7390         if (!t->actions)
7391                 goto nomem;
7392
7393         if (action_data_size_max) {
7394                 t->default_action_data = calloc(1, action_data_size_max);
7395                 if (!t->default_action_data)
7396                         goto nomem;
7397         }
7398
7399         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7400         if (!t->action_is_for_table_entries)
7401                 goto nomem;
7402
7403         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7404         if (!t->action_is_for_default_entry)
7405                 goto nomem;
7406
7407         /* Node initialization. */
7408         strcpy(t->name, name);
7409         if (args && args[0])
7410                 strcpy(t->args, args);
7411         t->type = type;
7412
7413         for (i = 0; i < params->n_fields; i++) {
7414                 struct rte_swx_match_field_params *field = &params->fields[i];
7415                 struct match_field *f = &t->fields[i];
7416
7417                 f->match_type = field->match_type;
7418                 f->field = header ?
7419                         header_field_parse(p, field->name, NULL) :
7420                         metadata_field_parse(p, field->name);
7421         }
7422         t->n_fields = params->n_fields;
7423         t->header = header;
7424
7425         for (i = 0; i < params->n_actions; i++) {
7426                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7427
7428                 if (params->action_is_for_table_entries)
7429                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7430                 if (params->action_is_for_default_entry)
7431                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7432
7433                 t->actions[i] = action_find(p, params->action_names[i]);
7434                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7435                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7436         }
7437         t->default_action = default_action;
7438         if (default_action->st)
7439                 memcpy(t->default_action_data,
7440                        params->default_action_data,
7441                        default_action->st->n_bits / 8);
7442         t->n_actions = params->n_actions;
7443         t->default_action_is_const = params->default_action_is_const;
7444         t->action_data_size_max = action_data_size_max;
7445
7446         t->size = size;
7447         t->id = p->n_tables;
7448
7449         /* Node add to tailq. */
7450         TAILQ_INSERT_TAIL(&p->tables, t, node);
7451         p->n_tables++;
7452
7453         return 0;
7454
7455 nomem:
7456         if (!t)
7457                 return -ENOMEM;
7458
7459         free(t->action_is_for_default_entry);
7460         free(t->action_is_for_table_entries);
7461         free(t->default_action_data);
7462         free(t->actions);
7463         free(t->fields);
7464         free(t);
7465
7466         return -ENOMEM;
7467 }
7468
7469 static struct rte_swx_table_params *
7470 table_params_get(struct table *table)
7471 {
7472         struct rte_swx_table_params *params;
7473         struct field *first, *last;
7474         uint8_t *key_mask;
7475         uint32_t key_size, key_offset, action_data_size, i;
7476
7477         /* Memory allocation. */
7478         params = calloc(1, sizeof(struct rte_swx_table_params));
7479         if (!params)
7480                 return NULL;
7481
7482         /* Find first (smallest offset) and last (biggest offset) match fields. */
7483         first = table->fields[0].field;
7484         last = table->fields[0].field;
7485
7486         for (i = 0; i < table->n_fields; i++) {
7487                 struct field *f = table->fields[i].field;
7488
7489                 if (f->offset < first->offset)
7490                         first = f;
7491
7492                 if (f->offset > last->offset)
7493                         last = f;
7494         }
7495
7496         /* Key offset and size. */
7497         key_offset = first->offset / 8;
7498         key_size = (last->offset + last->n_bits - first->offset) / 8;
7499
7500         /* Memory allocation. */
7501         key_mask = calloc(1, key_size);
7502         if (!key_mask) {
7503                 free(params);
7504                 return NULL;
7505         }
7506
7507         /* Key mask. */
7508         for (i = 0; i < table->n_fields; i++) {
7509                 struct field *f = table->fields[i].field;
7510                 uint32_t start = (f->offset - first->offset) / 8;
7511                 size_t size = f->n_bits / 8;
7512
7513                 memset(&key_mask[start], 0xFF, size);
7514         }
7515
7516         /* Action data size. */
7517         action_data_size = 0;
7518         for (i = 0; i < table->n_actions; i++) {
7519                 struct action *action = table->actions[i];
7520                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7521
7522                 if (ads > action_data_size)
7523                         action_data_size = ads;
7524         }
7525
7526         /* Fill in. */
7527         params->match_type = table->type->match_type;
7528         params->key_size = key_size;
7529         params->key_offset = key_offset;
7530         params->key_mask0 = key_mask;
7531         params->action_data_size = action_data_size;
7532         params->n_keys_max = table->size;
7533
7534         return params;
7535 }
7536
7537 static void
7538 table_params_free(struct rte_swx_table_params *params)
7539 {
7540         if (!params)
7541                 return;
7542
7543         free(params->key_mask0);
7544         free(params);
7545 }
7546
7547 static int
7548 table_stub_lkp(void *table __rte_unused,
7549                void *mailbox __rte_unused,
7550                uint8_t **key __rte_unused,
7551                uint64_t *action_id __rte_unused,
7552                uint8_t **action_data __rte_unused,
7553                int *hit)
7554 {
7555         *hit = 0;
7556         return 1; /* DONE. */
7557 }
7558
7559 static int
7560 table_build(struct rte_swx_pipeline *p)
7561 {
7562         uint32_t i;
7563
7564         /* Per pipeline: table statistics. */
7565         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7566         CHECK(p->table_stats, ENOMEM);
7567
7568         for (i = 0; i < p->n_tables; i++) {
7569                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7570                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7571         }
7572
7573         /* Per thread: table runt-time. */
7574         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7575                 struct thread *t = &p->threads[i];
7576                 struct table *table;
7577
7578                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7579                 CHECK(t->tables, ENOMEM);
7580
7581                 TAILQ_FOREACH(table, &p->tables, node) {
7582                         struct table_runtime *r = &t->tables[table->id];
7583
7584                         if (table->type) {
7585                                 uint64_t size;
7586
7587                                 size = table->type->ops.mailbox_size_get();
7588
7589                                 /* r->func. */
7590                                 r->func = table->type->ops.lkp;
7591
7592                                 /* r->mailbox. */
7593                                 if (size) {
7594                                         r->mailbox = calloc(1, size);
7595                                         CHECK(r->mailbox, ENOMEM);
7596                                 }
7597
7598                                 /* r->key. */
7599                                 r->key = table->header ?
7600                                         &t->structs[table->header->struct_id] :
7601                                         &t->structs[p->metadata_struct_id];
7602                         } else {
7603                                 r->func = table_stub_lkp;
7604                         }
7605                 }
7606         }
7607
7608         return 0;
7609 }
7610
7611 static void
7612 table_build_free(struct rte_swx_pipeline *p)
7613 {
7614         uint32_t i;
7615
7616         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7617                 struct thread *t = &p->threads[i];
7618                 uint32_t j;
7619
7620                 if (!t->tables)
7621                         continue;
7622
7623                 for (j = 0; j < p->n_tables; j++) {
7624                         struct table_runtime *r = &t->tables[j];
7625
7626                         free(r->mailbox);
7627                 }
7628
7629                 free(t->tables);
7630                 t->tables = NULL;
7631         }
7632
7633         if (p->table_stats) {
7634                 for (i = 0; i < p->n_tables; i++)
7635                         free(p->table_stats[i].n_pkts_action);
7636
7637                 free(p->table_stats);
7638         }
7639 }
7640
7641 static void
7642 table_free(struct rte_swx_pipeline *p)
7643 {
7644         table_build_free(p);
7645
7646         /* Tables. */
7647         for ( ; ; ) {
7648                 struct table *elem;
7649
7650                 elem = TAILQ_FIRST(&p->tables);
7651                 if (!elem)
7652                         break;
7653
7654                 TAILQ_REMOVE(&p->tables, elem, node);
7655                 free(elem->fields);
7656                 free(elem->actions);
7657                 free(elem->default_action_data);
7658                 free(elem);
7659         }
7660
7661         /* Table types. */
7662         for ( ; ; ) {
7663                 struct table_type *elem;
7664
7665                 elem = TAILQ_FIRST(&p->table_types);
7666                 if (!elem)
7667                         break;
7668
7669                 TAILQ_REMOVE(&p->table_types, elem, node);
7670                 free(elem);
7671         }
7672 }
7673
7674 /*
7675  * Selector.
7676  */
7677 static struct selector *
7678 selector_find(struct rte_swx_pipeline *p, const char *name)
7679 {
7680         struct selector *s;
7681
7682         TAILQ_FOREACH(s, &p->selectors, node)
7683                 if (strcmp(s->name, name) == 0)
7684                         return s;
7685
7686         return NULL;
7687 }
7688
7689 static struct selector *
7690 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7691 {
7692         struct selector *s = NULL;
7693
7694         TAILQ_FOREACH(s, &p->selectors, node)
7695                 if (s->id == id)
7696                         return s;
7697
7698         return NULL;
7699 }
7700
7701 static int
7702 selector_fields_check(struct rte_swx_pipeline *p,
7703                       struct rte_swx_pipeline_selector_params *params,
7704                       struct header **header)
7705 {
7706         struct header *h0 = NULL;
7707         struct field *hf, *mf;
7708         uint32_t i;
7709
7710         /* Return if no selector fields. */
7711         if (!params->n_selector_fields || !params->selector_field_names)
7712                 return -EINVAL;
7713
7714         /* Check that all the selector fields either belong to the same header
7715          * or are all meta-data fields.
7716          */
7717         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7718         mf = metadata_field_parse(p, params->selector_field_names[0]);
7719         if (!hf && !mf)
7720                 return -EINVAL;
7721
7722         for (i = 1; i < params->n_selector_fields; i++)
7723                 if (h0) {
7724                         struct header *h;
7725
7726                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7727                         if (!hf || (h->id != h0->id))
7728                                 return -EINVAL;
7729                 } else {
7730                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7731                         if (!mf)
7732                                 return -EINVAL;
7733                 }
7734
7735         /* Check that there are no duplicated match fields. */
7736         for (i = 0; i < params->n_selector_fields; i++) {
7737                 const char *field_name = params->selector_field_names[i];
7738                 uint32_t j;
7739
7740                 for (j = i + 1; j < params->n_selector_fields; j++)
7741                         if (!strcmp(params->selector_field_names[j], field_name))
7742                                 return -EINVAL;
7743         }
7744
7745         /* Return. */
7746         if (header)
7747                 *header = h0;
7748
7749         return 0;
7750 }
7751
7752 int
7753 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7754                                  const char *name,
7755                                  struct rte_swx_pipeline_selector_params *params)
7756 {
7757         struct selector *s;
7758         struct header *selector_header = NULL;
7759         struct field *group_id_field, *member_id_field;
7760         uint32_t i;
7761         int status = 0;
7762
7763         CHECK(p, EINVAL);
7764
7765         CHECK_NAME(name, EINVAL);
7766         CHECK(!table_find(p, name), EEXIST);
7767         CHECK(!selector_find(p, name), EEXIST);
7768         CHECK(!learner_find(p, name), EEXIST);
7769
7770         CHECK(params, EINVAL);
7771
7772         CHECK_NAME(params->group_id_field_name, EINVAL);
7773         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7774         CHECK(group_id_field, EINVAL);
7775
7776         for (i = 0; i < params->n_selector_fields; i++) {
7777                 const char *field_name = params->selector_field_names[i];
7778
7779                 CHECK_NAME(field_name, EINVAL);
7780         }
7781         status = selector_fields_check(p, params, &selector_header);
7782         if (status)
7783                 return status;
7784
7785         CHECK_NAME(params->member_id_field_name, EINVAL);
7786         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7787         CHECK(member_id_field, EINVAL);
7788
7789         CHECK(params->n_groups_max, EINVAL);
7790
7791         CHECK(params->n_members_per_group_max, EINVAL);
7792
7793         /* Memory allocation. */
7794         s = calloc(1, sizeof(struct selector));
7795         if (!s) {
7796                 status = -ENOMEM;
7797                 goto error;
7798         }
7799
7800         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7801         if (!s->selector_fields) {
7802                 status = -ENOMEM;
7803                 goto error;
7804         }
7805
7806         /* Node initialization. */
7807         strcpy(s->name, name);
7808
7809         s->group_id_field = group_id_field;
7810
7811         for (i = 0; i < params->n_selector_fields; i++) {
7812                 const char *field_name = params->selector_field_names[i];
7813
7814                 s->selector_fields[i] = selector_header ?
7815                         header_field_parse(p, field_name, NULL) :
7816                         metadata_field_parse(p, field_name);
7817         }
7818
7819         s->n_selector_fields = params->n_selector_fields;
7820
7821         s->selector_header = selector_header;
7822
7823         s->member_id_field = member_id_field;
7824
7825         s->n_groups_max = params->n_groups_max;
7826
7827         s->n_members_per_group_max = params->n_members_per_group_max;
7828
7829         s->id = p->n_selectors;
7830
7831         /* Node add to tailq. */
7832         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7833         p->n_selectors++;
7834
7835         return 0;
7836
7837 error:
7838         if (!s)
7839                 return status;
7840
7841         free(s->selector_fields);
7842
7843         free(s);
7844
7845         return status;
7846 }
7847
7848 static void
7849 selector_params_free(struct rte_swx_table_selector_params *params)
7850 {
7851         if (!params)
7852                 return;
7853
7854         free(params->selector_mask);
7855
7856         free(params);
7857 }
7858
7859 static struct rte_swx_table_selector_params *
7860 selector_table_params_get(struct selector *s)
7861 {
7862         struct rte_swx_table_selector_params *params = NULL;
7863         struct field *first, *last;
7864         uint32_t i;
7865
7866         /* Memory allocation. */
7867         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7868         if (!params)
7869                 goto error;
7870
7871         /* Group ID. */
7872         params->group_id_offset = s->group_id_field->offset / 8;
7873
7874         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7875         first = s->selector_fields[0];
7876         last = s->selector_fields[0];
7877
7878         for (i = 0; i < s->n_selector_fields; i++) {
7879                 struct field *f = s->selector_fields[i];
7880
7881                 if (f->offset < first->offset)
7882                         first = f;
7883
7884                 if (f->offset > last->offset)
7885                         last = f;
7886         }
7887
7888         /* Selector offset and size. */
7889         params->selector_offset = first->offset / 8;
7890         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7891
7892         /* Memory allocation. */
7893         params->selector_mask = calloc(1, params->selector_size);
7894         if (!params->selector_mask)
7895                 goto error;
7896
7897         /* Selector mask. */
7898         for (i = 0; i < s->n_selector_fields; i++) {
7899                 struct field *f = s->selector_fields[i];
7900                 uint32_t start = (f->offset - first->offset) / 8;
7901                 size_t size = f->n_bits / 8;
7902
7903                 memset(&params->selector_mask[start], 0xFF, size);
7904         }
7905
7906         /* Member ID. */
7907         params->member_id_offset = s->member_id_field->offset / 8;
7908
7909         /* Maximum number of groups. */
7910         params->n_groups_max = s->n_groups_max;
7911
7912         /* Maximum number of members per group. */
7913         params->n_members_per_group_max = s->n_members_per_group_max;
7914
7915         return params;
7916
7917 error:
7918         selector_params_free(params);
7919         return NULL;
7920 }
7921
7922 static void
7923 selector_build_free(struct rte_swx_pipeline *p)
7924 {
7925         uint32_t i;
7926
7927         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7928                 struct thread *t = &p->threads[i];
7929                 uint32_t j;
7930
7931                 if (!t->selectors)
7932                         continue;
7933
7934                 for (j = 0; j < p->n_selectors; j++) {
7935                         struct selector_runtime *r = &t->selectors[j];
7936
7937                         free(r->mailbox);
7938                 }
7939
7940                 free(t->selectors);
7941                 t->selectors = NULL;
7942         }
7943
7944         free(p->selector_stats);
7945         p->selector_stats = NULL;
7946 }
7947
7948 static int
7949 selector_build(struct rte_swx_pipeline *p)
7950 {
7951         uint32_t i;
7952         int status = 0;
7953
7954         /* Per pipeline: selector statistics. */
7955         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7956         if (!p->selector_stats) {
7957                 status = -ENOMEM;
7958                 goto error;
7959         }
7960
7961         /* Per thread: selector run-time. */
7962         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7963                 struct thread *t = &p->threads[i];
7964                 struct selector *s;
7965
7966                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7967                 if (!t->selectors) {
7968                         status = -ENOMEM;
7969                         goto error;
7970                 }
7971
7972                 TAILQ_FOREACH(s, &p->selectors, node) {
7973                         struct selector_runtime *r = &t->selectors[s->id];
7974                         uint64_t size;
7975
7976                         /* r->mailbox. */
7977                         size = rte_swx_table_selector_mailbox_size_get();
7978                         if (size) {
7979                                 r->mailbox = calloc(1, size);
7980                                 if (!r->mailbox) {
7981                                         status = -ENOMEM;
7982                                         goto error;
7983                                 }
7984                         }
7985
7986                         /* r->group_id_buffer. */
7987                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
7988
7989                         /* r->selector_buffer. */
7990                         r->selector_buffer = s->selector_header ?
7991                                 &t->structs[s->selector_header->struct_id] :
7992                                 &t->structs[p->metadata_struct_id];
7993
7994                         /* r->member_id_buffer. */
7995                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
7996                 }
7997         }
7998
7999         return 0;
8000
8001 error:
8002         selector_build_free(p);
8003         return status;
8004 }
8005
8006 static void
8007 selector_free(struct rte_swx_pipeline *p)
8008 {
8009         selector_build_free(p);
8010
8011         /* Selector tables. */
8012         for ( ; ; ) {
8013                 struct selector *elem;
8014
8015                 elem = TAILQ_FIRST(&p->selectors);
8016                 if (!elem)
8017                         break;
8018
8019                 TAILQ_REMOVE(&p->selectors, elem, node);
8020                 free(elem->selector_fields);
8021                 free(elem);
8022         }
8023 }
8024
8025 /*
8026  * Learner table.
8027  */
8028 static struct learner *
8029 learner_find(struct rte_swx_pipeline *p, const char *name)
8030 {
8031         struct learner *l;
8032
8033         TAILQ_FOREACH(l, &p->learners, node)
8034                 if (!strcmp(l->name, name))
8035                         return l;
8036
8037         return NULL;
8038 }
8039
8040 static struct learner *
8041 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8042 {
8043         struct learner *l = NULL;
8044
8045         TAILQ_FOREACH(l, &p->learners, node)
8046                 if (l->id == id)
8047                         return l;
8048
8049         return NULL;
8050 }
8051
8052 static int
8053 learner_match_fields_check(struct rte_swx_pipeline *p,
8054                            struct rte_swx_pipeline_learner_params *params,
8055                            struct header **header)
8056 {
8057         struct header *h0 = NULL;
8058         struct field *hf, *mf;
8059         uint32_t i;
8060
8061         /* Return if no match fields. */
8062         if (!params->n_fields || !params->field_names)
8063                 return -EINVAL;
8064
8065         /* Check that all the match fields either belong to the same header
8066          * or are all meta-data fields.
8067          */
8068         hf = header_field_parse(p, params->field_names[0], &h0);
8069         mf = metadata_field_parse(p, params->field_names[0]);
8070         if (!hf && !mf)
8071                 return -EINVAL;
8072
8073         for (i = 1; i < params->n_fields; i++)
8074                 if (h0) {
8075                         struct header *h;
8076
8077                         hf = header_field_parse(p, params->field_names[i], &h);
8078                         if (!hf || (h->id != h0->id))
8079                                 return -EINVAL;
8080                 } else {
8081                         mf = metadata_field_parse(p, params->field_names[i]);
8082                         if (!mf)
8083                                 return -EINVAL;
8084                 }
8085
8086         /* Check that there are no duplicated match fields. */
8087         for (i = 0; i < params->n_fields; i++) {
8088                 const char *field_name = params->field_names[i];
8089                 uint32_t j;
8090
8091                 for (j = i + 1; j < params->n_fields; j++)
8092                         if (!strcmp(params->field_names[j], field_name))
8093                                 return -EINVAL;
8094         }
8095
8096         /* Return. */
8097         if (header)
8098                 *header = h0;
8099
8100         return 0;
8101 }
8102
8103 static int
8104 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8105 {
8106         struct struct_type *mst = p->metadata_st, *ast = a->st;
8107         struct field *mf, *af;
8108         uint32_t mf_pos, i;
8109
8110         if (!ast) {
8111                 if (mf_name)
8112                         return -EINVAL;
8113
8114                 return 0;
8115         }
8116
8117         /* Check that mf_name is the name of a valid meta-data field. */
8118         CHECK_NAME(mf_name, EINVAL);
8119         mf = metadata_field_parse(p, mf_name);
8120         CHECK(mf, EINVAL);
8121
8122         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8123          * all the action arguments.
8124          */
8125         mf_pos = mf - mst->fields;
8126         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8127
8128         /* Check that the size of each of the identified meta-data fields matches exactly the size
8129          * of the corresponding action argument.
8130          */
8131         for (i = 0; i < ast->n_fields; i++) {
8132                 mf = &mst->fields[mf_pos + i];
8133                 af = &ast->fields[i];
8134
8135                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8136         }
8137
8138         return 0;
8139 }
8140
8141 static int
8142 learner_action_learning_check(struct rte_swx_pipeline *p,
8143                               struct action *action,
8144                               const char **action_names,
8145                               uint32_t n_actions)
8146 {
8147         uint32_t i;
8148
8149         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8150          * the action passed as argument to the "learn" instruction) is also enabled for the
8151          * current learner table.
8152          */
8153         for (i = 0; i < action->n_instructions; i++) {
8154                 struct instruction *instr = &action->instructions[i];
8155                 uint32_t found = 0, j;
8156
8157                 if (instr->type != INSTR_LEARNER_LEARN)
8158                         continue;
8159
8160                 for (j = 0; j < n_actions; j++) {
8161                         struct action *a;
8162
8163                         a = action_find(p, action_names[j]);
8164                         if (!a)
8165                                 return -EINVAL;
8166
8167                         if (a->id == instr->learn.action_id)
8168                                 found = 1;
8169                 }
8170
8171                 if (!found)
8172                         return -EINVAL;
8173         }
8174
8175         return 0;
8176 }
8177
8178 int
8179 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8180                               const char *name,
8181                               struct rte_swx_pipeline_learner_params *params,
8182                               uint32_t size,
8183                               uint32_t timeout)
8184 {
8185         struct learner *l = NULL;
8186         struct action *default_action;
8187         struct header *header = NULL;
8188         uint32_t action_data_size_max = 0, i;
8189         int status = 0;
8190
8191         CHECK(p, EINVAL);
8192
8193         CHECK_NAME(name, EINVAL);
8194         CHECK(!table_find(p, name), EEXIST);
8195         CHECK(!selector_find(p, name), EEXIST);
8196         CHECK(!learner_find(p, name), EEXIST);
8197
8198         CHECK(params, EINVAL);
8199
8200         /* Match checks. */
8201         status = learner_match_fields_check(p, params, &header);
8202         if (status)
8203                 return status;
8204
8205         /* Action checks. */
8206         CHECK(params->n_actions, EINVAL);
8207         CHECK(params->action_names, EINVAL);
8208         for (i = 0; i < params->n_actions; i++) {
8209                 const char *action_name = params->action_names[i];
8210                 struct action *a;
8211                 uint32_t action_data_size;
8212                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8213
8214                 CHECK_NAME(action_name, EINVAL);
8215
8216                 a = action_find(p, action_name);
8217                 CHECK(a, EINVAL);
8218
8219                 status = learner_action_learning_check(p,
8220                                                        a,
8221                                                        params->action_names,
8222                                                        params->n_actions);
8223                 if (status)
8224                         return status;
8225
8226                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8227                 if (action_data_size > action_data_size_max)
8228                         action_data_size_max = action_data_size;
8229
8230                 if (params->action_is_for_table_entries)
8231                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8232                 if (params->action_is_for_default_entry)
8233                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8234                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8235         }
8236
8237         CHECK_NAME(params->default_action_name, EINVAL);
8238         for (i = 0; i < p->n_actions; i++)
8239                 if (!strcmp(params->action_names[i],
8240                             params->default_action_name))
8241                         break;
8242         CHECK(i < params->n_actions, EINVAL);
8243         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8244               EINVAL);
8245
8246         default_action = action_find(p, params->default_action_name);
8247         CHECK((default_action->st && params->default_action_data) ||
8248               !params->default_action_data, EINVAL);
8249
8250         /* Any other checks. */
8251         CHECK(size, EINVAL);
8252         CHECK(timeout, EINVAL);
8253
8254         /* Memory allocation. */
8255         l = calloc(1, sizeof(struct learner));
8256         if (!l)
8257                 goto nomem;
8258
8259         l->fields = calloc(params->n_fields, sizeof(struct field *));
8260         if (!l->fields)
8261                 goto nomem;
8262
8263         l->actions = calloc(params->n_actions, sizeof(struct action *));
8264         if (!l->actions)
8265                 goto nomem;
8266
8267         if (action_data_size_max) {
8268                 l->default_action_data = calloc(1, action_data_size_max);
8269                 if (!l->default_action_data)
8270                         goto nomem;
8271         }
8272
8273         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8274         if (!l->action_is_for_table_entries)
8275                 goto nomem;
8276
8277         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8278         if (!l->action_is_for_default_entry)
8279                 goto nomem;
8280
8281         /* Node initialization. */
8282         strcpy(l->name, name);
8283
8284         for (i = 0; i < params->n_fields; i++) {
8285                 const char *field_name = params->field_names[i];
8286
8287                 l->fields[i] = header ?
8288                         header_field_parse(p, field_name, NULL) :
8289                         metadata_field_parse(p, field_name);
8290         }
8291
8292         l->n_fields = params->n_fields;
8293
8294         l->header = header;
8295
8296         for (i = 0; i < params->n_actions; i++) {
8297                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8298
8299                 if (params->action_is_for_table_entries)
8300                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8301                 if (params->action_is_for_default_entry)
8302                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8303
8304                 l->actions[i] = action_find(p, params->action_names[i]);
8305                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8306                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8307         }
8308
8309         l->default_action = default_action;
8310
8311         if (default_action->st)
8312                 memcpy(l->default_action_data,
8313                        params->default_action_data,
8314                        default_action->st->n_bits / 8);
8315
8316         l->n_actions = params->n_actions;
8317
8318         l->default_action_is_const = params->default_action_is_const;
8319
8320         l->action_data_size_max = action_data_size_max;
8321
8322         l->size = size;
8323
8324         l->timeout = timeout;
8325
8326         l->id = p->n_learners;
8327
8328         /* Node add to tailq. */
8329         TAILQ_INSERT_TAIL(&p->learners, l, node);
8330         p->n_learners++;
8331
8332         return 0;
8333
8334 nomem:
8335         if (!l)
8336                 return -ENOMEM;
8337
8338         free(l->action_is_for_default_entry);
8339         free(l->action_is_for_table_entries);
8340         free(l->default_action_data);
8341         free(l->actions);
8342         free(l->fields);
8343         free(l);
8344
8345         return -ENOMEM;
8346 }
8347
8348 static void
8349 learner_params_free(struct rte_swx_table_learner_params *params)
8350 {
8351         if (!params)
8352                 return;
8353
8354         free(params->key_mask0);
8355
8356         free(params);
8357 }
8358
8359 static struct rte_swx_table_learner_params *
8360 learner_params_get(struct learner *l)
8361 {
8362         struct rte_swx_table_learner_params *params = NULL;
8363         struct field *first, *last;
8364         uint32_t i;
8365
8366         /* Memory allocation. */
8367         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8368         if (!params)
8369                 goto error;
8370
8371         /* Find first (smallest offset) and last (biggest offset) match fields. */
8372         first = l->fields[0];
8373         last = l->fields[0];
8374
8375         for (i = 0; i < l->n_fields; i++) {
8376                 struct field *f = l->fields[i];
8377
8378                 if (f->offset < first->offset)
8379                         first = f;
8380
8381                 if (f->offset > last->offset)
8382                         last = f;
8383         }
8384
8385         /* Key offset and size. */
8386         params->key_offset = first->offset / 8;
8387         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8388
8389         /* Memory allocation. */
8390         params->key_mask0 = calloc(1, params->key_size);
8391         if (!params->key_mask0)
8392                 goto error;
8393
8394         /* Key mask. */
8395         for (i = 0; i < l->n_fields; i++) {
8396                 struct field *f = l->fields[i];
8397                 uint32_t start = (f->offset - first->offset) / 8;
8398                 size_t size = f->n_bits / 8;
8399
8400                 memset(&params->key_mask0[start], 0xFF, size);
8401         }
8402
8403         /* Action data size. */
8404         params->action_data_size = l->action_data_size_max;
8405
8406         /* Maximum number of keys. */
8407         params->n_keys_max = l->size;
8408
8409         /* Timeout. */
8410         params->key_timeout = l->timeout;
8411
8412         return params;
8413
8414 error:
8415         learner_params_free(params);
8416         return NULL;
8417 }
8418
8419 static void
8420 learner_build_free(struct rte_swx_pipeline *p)
8421 {
8422         uint32_t i;
8423
8424         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8425                 struct thread *t = &p->threads[i];
8426                 uint32_t j;
8427
8428                 if (!t->learners)
8429                         continue;
8430
8431                 for (j = 0; j < p->n_learners; j++) {
8432                         struct learner_runtime *r = &t->learners[j];
8433
8434                         free(r->mailbox);
8435                 }
8436
8437                 free(t->learners);
8438                 t->learners = NULL;
8439         }
8440
8441         if (p->learner_stats) {
8442                 for (i = 0; i < p->n_learners; i++)
8443                         free(p->learner_stats[i].n_pkts_action);
8444
8445                 free(p->learner_stats);
8446         }
8447 }
8448
8449 static int
8450 learner_build(struct rte_swx_pipeline *p)
8451 {
8452         uint32_t i;
8453         int status = 0;
8454
8455         /* Per pipeline: learner statistics. */
8456         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8457         CHECK(p->learner_stats, ENOMEM);
8458
8459         for (i = 0; i < p->n_learners; i++) {
8460                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8461                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8462         }
8463
8464         /* Per thread: learner run-time. */
8465         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8466                 struct thread *t = &p->threads[i];
8467                 struct learner *l;
8468
8469                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8470                 if (!t->learners) {
8471                         status = -ENOMEM;
8472                         goto error;
8473                 }
8474
8475                 TAILQ_FOREACH(l, &p->learners, node) {
8476                         struct learner_runtime *r = &t->learners[l->id];
8477                         uint64_t size;
8478
8479                         /* r->mailbox. */
8480                         size = rte_swx_table_learner_mailbox_size_get();
8481                         if (size) {
8482                                 r->mailbox = calloc(1, size);
8483                                 if (!r->mailbox) {
8484                                         status = -ENOMEM;
8485                                         goto error;
8486                                 }
8487                         }
8488
8489                         /* r->key. */
8490                         r->key = l->header ?
8491                                 &t->structs[l->header->struct_id] :
8492                                 &t->structs[p->metadata_struct_id];
8493                 }
8494         }
8495
8496         return 0;
8497
8498 error:
8499         learner_build_free(p);
8500         return status;
8501 }
8502
8503 static void
8504 learner_free(struct rte_swx_pipeline *p)
8505 {
8506         learner_build_free(p);
8507
8508         /* Learner tables. */
8509         for ( ; ; ) {
8510                 struct learner *l;
8511
8512                 l = TAILQ_FIRST(&p->learners);
8513                 if (!l)
8514                         break;
8515
8516                 TAILQ_REMOVE(&p->learners, l, node);
8517                 free(l->fields);
8518                 free(l->actions);
8519                 free(l->default_action_data);
8520                 free(l);
8521         }
8522 }
8523
8524 /*
8525  * Table state.
8526  */
8527 static int
8528 table_state_build(struct rte_swx_pipeline *p)
8529 {
8530         struct table *table;
8531         struct selector *s;
8532         struct learner *l;
8533
8534         p->table_state = calloc(p->n_tables + p->n_selectors,
8535                                 sizeof(struct rte_swx_table_state));
8536         CHECK(p->table_state, ENOMEM);
8537
8538         TAILQ_FOREACH(table, &p->tables, node) {
8539                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8540
8541                 if (table->type) {
8542                         struct rte_swx_table_params *params;
8543
8544                         /* ts->obj. */
8545                         params = table_params_get(table);
8546                         CHECK(params, ENOMEM);
8547
8548                         ts->obj = table->type->ops.create(params,
8549                                 NULL,
8550                                 table->args,
8551                                 p->numa_node);
8552
8553                         table_params_free(params);
8554                         CHECK(ts->obj, ENODEV);
8555                 }
8556
8557                 /* ts->default_action_data. */
8558                 if (table->action_data_size_max) {
8559                         ts->default_action_data =
8560                                 malloc(table->action_data_size_max);
8561                         CHECK(ts->default_action_data, ENOMEM);
8562
8563                         memcpy(ts->default_action_data,
8564                                table->default_action_data,
8565                                table->action_data_size_max);
8566                 }
8567
8568                 /* ts->default_action_id. */
8569                 ts->default_action_id = table->default_action->id;
8570         }
8571
8572         TAILQ_FOREACH(s, &p->selectors, node) {
8573                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8574                 struct rte_swx_table_selector_params *params;
8575
8576                 /* ts->obj. */
8577                 params = selector_table_params_get(s);
8578                 CHECK(params, ENOMEM);
8579
8580                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8581
8582                 selector_params_free(params);
8583                 CHECK(ts->obj, ENODEV);
8584         }
8585
8586         TAILQ_FOREACH(l, &p->learners, node) {
8587                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8588                         p->n_selectors + l->id];
8589                 struct rte_swx_table_learner_params *params;
8590
8591                 /* ts->obj. */
8592                 params = learner_params_get(l);
8593                 CHECK(params, ENOMEM);
8594
8595                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8596                 learner_params_free(params);
8597                 CHECK(ts->obj, ENODEV);
8598
8599                 /* ts->default_action_data. */
8600                 if (l->action_data_size_max) {
8601                         ts->default_action_data = malloc(l->action_data_size_max);
8602                         CHECK(ts->default_action_data, ENOMEM);
8603
8604                         memcpy(ts->default_action_data,
8605                                l->default_action_data,
8606                                l->action_data_size_max);
8607                 }
8608
8609                 /* ts->default_action_id. */
8610                 ts->default_action_id = l->default_action->id;
8611         }
8612
8613         return 0;
8614 }
8615
8616 static void
8617 table_state_build_free(struct rte_swx_pipeline *p)
8618 {
8619         uint32_t i;
8620
8621         if (!p->table_state)
8622                 return;
8623
8624         for (i = 0; i < p->n_tables; i++) {
8625                 struct rte_swx_table_state *ts = &p->table_state[i];
8626                 struct table *table = table_find_by_id(p, i);
8627
8628                 /* ts->obj. */
8629                 if (table->type && ts->obj)
8630                         table->type->ops.free(ts->obj);
8631
8632                 /* ts->default_action_data. */
8633                 free(ts->default_action_data);
8634         }
8635
8636         for (i = 0; i < p->n_selectors; i++) {
8637                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8638
8639                 /* ts->obj. */
8640                 if (ts->obj)
8641                         rte_swx_table_selector_free(ts->obj);
8642         }
8643
8644         for (i = 0; i < p->n_learners; i++) {
8645                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8646
8647                 /* ts->obj. */
8648                 if (ts->obj)
8649                         rte_swx_table_learner_free(ts->obj);
8650
8651                 /* ts->default_action_data. */
8652                 free(ts->default_action_data);
8653         }
8654
8655         free(p->table_state);
8656         p->table_state = NULL;
8657 }
8658
8659 static void
8660 table_state_free(struct rte_swx_pipeline *p)
8661 {
8662         table_state_build_free(p);
8663 }
8664
8665 /*
8666  * Register array.
8667  */
8668 static struct regarray *
8669 regarray_find(struct rte_swx_pipeline *p, const char *name)
8670 {
8671         struct regarray *elem;
8672
8673         TAILQ_FOREACH(elem, &p->regarrays, node)
8674                 if (!strcmp(elem->name, name))
8675                         return elem;
8676
8677         return NULL;
8678 }
8679
8680 static struct regarray *
8681 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8682 {
8683         struct regarray *elem = NULL;
8684
8685         TAILQ_FOREACH(elem, &p->regarrays, node)
8686                 if (elem->id == id)
8687                         return elem;
8688
8689         return NULL;
8690 }
8691
8692 int
8693 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8694                               const char *name,
8695                               uint32_t size,
8696                               uint64_t init_val)
8697 {
8698         struct regarray *r;
8699
8700         CHECK(p, EINVAL);
8701
8702         CHECK_NAME(name, EINVAL);
8703         CHECK(!regarray_find(p, name), EEXIST);
8704
8705         CHECK(size, EINVAL);
8706         size = rte_align32pow2(size);
8707
8708         /* Memory allocation. */
8709         r = calloc(1, sizeof(struct regarray));
8710         CHECK(r, ENOMEM);
8711
8712         /* Node initialization. */
8713         strcpy(r->name, name);
8714         r->init_val = init_val;
8715         r->size = size;
8716         r->id = p->n_regarrays;
8717
8718         /* Node add to tailq. */
8719         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8720         p->n_regarrays++;
8721
8722         return 0;
8723 }
8724
8725 static int
8726 regarray_build(struct rte_swx_pipeline *p)
8727 {
8728         struct regarray *regarray;
8729
8730         if (!p->n_regarrays)
8731                 return 0;
8732
8733         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8734         CHECK(p->regarray_runtime, ENOMEM);
8735
8736         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8737                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8738                 uint32_t i;
8739
8740                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8741                                          RTE_CACHE_LINE_SIZE,
8742                                          p->numa_node);
8743                 CHECK(r->regarray, ENOMEM);
8744
8745                 if (regarray->init_val)
8746                         for (i = 0; i < regarray->size; i++)
8747                                 r->regarray[i] = regarray->init_val;
8748
8749                 r->size_mask = regarray->size - 1;
8750         }
8751
8752         return 0;
8753 }
8754
8755 static void
8756 regarray_build_free(struct rte_swx_pipeline *p)
8757 {
8758         uint32_t i;
8759
8760         if (!p->regarray_runtime)
8761                 return;
8762
8763         for (i = 0; i < p->n_regarrays; i++) {
8764                 struct regarray *regarray = regarray_find_by_id(p, i);
8765                 struct regarray_runtime *r = &p->regarray_runtime[i];
8766
8767                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8768         }
8769
8770         free(p->regarray_runtime);
8771         p->regarray_runtime = NULL;
8772 }
8773
8774 static void
8775 regarray_free(struct rte_swx_pipeline *p)
8776 {
8777         regarray_build_free(p);
8778
8779         for ( ; ; ) {
8780                 struct regarray *elem;
8781
8782                 elem = TAILQ_FIRST(&p->regarrays);
8783                 if (!elem)
8784                         break;
8785
8786                 TAILQ_REMOVE(&p->regarrays, elem, node);
8787                 free(elem);
8788         }
8789 }
8790
8791 /*
8792  * Meter array.
8793  */
8794 static struct meter_profile *
8795 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8796 {
8797         struct meter_profile *elem;
8798
8799         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8800                 if (!strcmp(elem->name, name))
8801                         return elem;
8802
8803         return NULL;
8804 }
8805
8806 static struct metarray *
8807 metarray_find(struct rte_swx_pipeline *p, const char *name)
8808 {
8809         struct metarray *elem;
8810
8811         TAILQ_FOREACH(elem, &p->metarrays, node)
8812                 if (!strcmp(elem->name, name))
8813                         return elem;
8814
8815         return NULL;
8816 }
8817
8818 static struct metarray *
8819 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8820 {
8821         struct metarray *elem = NULL;
8822
8823         TAILQ_FOREACH(elem, &p->metarrays, node)
8824                 if (elem->id == id)
8825                         return elem;
8826
8827         return NULL;
8828 }
8829
8830 int
8831 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8832                                  const char *name,
8833                                  uint32_t size)
8834 {
8835         struct metarray *m;
8836
8837         CHECK(p, EINVAL);
8838
8839         CHECK_NAME(name, EINVAL);
8840         CHECK(!metarray_find(p, name), EEXIST);
8841
8842         CHECK(size, EINVAL);
8843         size = rte_align32pow2(size);
8844
8845         /* Memory allocation. */
8846         m = calloc(1, sizeof(struct metarray));
8847         CHECK(m, ENOMEM);
8848
8849         /* Node initialization. */
8850         strcpy(m->name, name);
8851         m->size = size;
8852         m->id = p->n_metarrays;
8853
8854         /* Node add to tailq. */
8855         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8856         p->n_metarrays++;
8857
8858         return 0;
8859 }
8860
8861 struct meter_profile meter_profile_default = {
8862         .node = {0},
8863         .name = "",
8864         .params = {0},
8865
8866         .profile = {
8867                 .cbs = 10000,
8868                 .pbs = 10000,
8869                 .cir_period = 1,
8870                 .cir_bytes_per_period = 1,
8871                 .pir_period = 1,
8872                 .pir_bytes_per_period = 1,
8873         },
8874
8875         .n_users = 0,
8876 };
8877
8878 static void
8879 meter_init(struct meter *m)
8880 {
8881         memset(m, 0, sizeof(struct meter));
8882         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8883         m->profile = &meter_profile_default;
8884         m->color_mask = RTE_COLOR_GREEN;
8885
8886         meter_profile_default.n_users++;
8887 }
8888
8889 static int
8890 metarray_build(struct rte_swx_pipeline *p)
8891 {
8892         struct metarray *m;
8893
8894         if (!p->n_metarrays)
8895                 return 0;
8896
8897         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8898         CHECK(p->metarray_runtime, ENOMEM);
8899
8900         TAILQ_FOREACH(m, &p->metarrays, node) {
8901                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8902                 uint32_t i;
8903
8904                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8905                                          RTE_CACHE_LINE_SIZE,
8906                                          p->numa_node);
8907                 CHECK(r->metarray, ENOMEM);
8908
8909                 for (i = 0; i < m->size; i++)
8910                         meter_init(&r->metarray[i]);
8911
8912                 r->size_mask = m->size - 1;
8913         }
8914
8915         return 0;
8916 }
8917
8918 static void
8919 metarray_build_free(struct rte_swx_pipeline *p)
8920 {
8921         uint32_t i;
8922
8923         if (!p->metarray_runtime)
8924                 return;
8925
8926         for (i = 0; i < p->n_metarrays; i++) {
8927                 struct metarray *m = metarray_find_by_id(p, i);
8928                 struct metarray_runtime *r = &p->metarray_runtime[i];
8929
8930                 env_free(r->metarray, m->size * sizeof(struct meter));
8931         }
8932
8933         free(p->metarray_runtime);
8934         p->metarray_runtime = NULL;
8935 }
8936
8937 static void
8938 metarray_free(struct rte_swx_pipeline *p)
8939 {
8940         metarray_build_free(p);
8941
8942         /* Meter arrays. */
8943         for ( ; ; ) {
8944                 struct metarray *elem;
8945
8946                 elem = TAILQ_FIRST(&p->metarrays);
8947                 if (!elem)
8948                         break;
8949
8950                 TAILQ_REMOVE(&p->metarrays, elem, node);
8951                 free(elem);
8952         }
8953
8954         /* Meter profiles. */
8955         for ( ; ; ) {
8956                 struct meter_profile *elem;
8957
8958                 elem = TAILQ_FIRST(&p->meter_profiles);
8959                 if (!elem)
8960                         break;
8961
8962                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8963                 free(elem);
8964         }
8965 }
8966
8967 /*
8968  * Pipeline.
8969  */
8970 int
8971 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8972 {
8973         struct rte_swx_pipeline *pipeline;
8974
8975         /* Check input parameters. */
8976         CHECK(p, EINVAL);
8977
8978         /* Memory allocation. */
8979         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8980         CHECK(pipeline, ENOMEM);
8981
8982         /* Initialization. */
8983         TAILQ_INIT(&pipeline->struct_types);
8984         TAILQ_INIT(&pipeline->port_in_types);
8985         TAILQ_INIT(&pipeline->ports_in);
8986         TAILQ_INIT(&pipeline->port_out_types);
8987         TAILQ_INIT(&pipeline->ports_out);
8988         TAILQ_INIT(&pipeline->extern_types);
8989         TAILQ_INIT(&pipeline->extern_objs);
8990         TAILQ_INIT(&pipeline->extern_funcs);
8991         TAILQ_INIT(&pipeline->headers);
8992         TAILQ_INIT(&pipeline->actions);
8993         TAILQ_INIT(&pipeline->table_types);
8994         TAILQ_INIT(&pipeline->tables);
8995         TAILQ_INIT(&pipeline->selectors);
8996         TAILQ_INIT(&pipeline->learners);
8997         TAILQ_INIT(&pipeline->regarrays);
8998         TAILQ_INIT(&pipeline->meter_profiles);
8999         TAILQ_INIT(&pipeline->metarrays);
9000
9001         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9002         pipeline->numa_node = numa_node;
9003
9004         *p = pipeline;
9005         return 0;
9006 }
9007
9008 void
9009 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9010 {
9011         void *lib;
9012
9013         if (!p)
9014                 return;
9015
9016         lib = p->lib;
9017
9018         free(p->instruction_data);
9019         free(p->instructions);
9020
9021         metarray_free(p);
9022         regarray_free(p);
9023         table_state_free(p);
9024         learner_free(p);
9025         selector_free(p);
9026         table_free(p);
9027         action_free(p);
9028         instruction_table_free(p);
9029         metadata_free(p);
9030         header_free(p);
9031         extern_func_free(p);
9032         extern_obj_free(p);
9033         port_out_free(p);
9034         port_in_free(p);
9035         struct_free(p);
9036
9037         free(p);
9038
9039         if (lib)
9040                 dlclose(lib);
9041 }
9042
9043 int
9044 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9045                                      const char **instructions,
9046                                      uint32_t n_instructions)
9047 {
9048         int err;
9049         uint32_t i;
9050
9051         err = instruction_config(p, NULL, instructions, n_instructions);
9052         if (err)
9053                 return err;
9054
9055         /* Thread instruction pointer reset. */
9056         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9057                 struct thread *t = &p->threads[i];
9058
9059                 thread_ip_reset(p, t);
9060         }
9061
9062         return 0;
9063 }
9064
9065 static int
9066 pipeline_compile(struct rte_swx_pipeline *p);
9067
9068 int
9069 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9070 {
9071         int status;
9072
9073         CHECK(p, EINVAL);
9074         CHECK(p->build_done == 0, EEXIST);
9075
9076         status = port_in_build(p);
9077         if (status)
9078                 goto error;
9079
9080         status = port_out_build(p);
9081         if (status)
9082                 goto error;
9083
9084         status = struct_build(p);
9085         if (status)
9086                 goto error;
9087
9088         status = extern_obj_build(p);
9089         if (status)
9090                 goto error;
9091
9092         status = extern_func_build(p);
9093         if (status)
9094                 goto error;
9095
9096         status = header_build(p);
9097         if (status)
9098                 goto error;
9099
9100         status = metadata_build(p);
9101         if (status)
9102                 goto error;
9103
9104         status = instruction_table_build(p);
9105         if (status)
9106                 goto error;
9107
9108         status = action_build(p);
9109         if (status)
9110                 goto error;
9111
9112         status = table_build(p);
9113         if (status)
9114                 goto error;
9115
9116         status = selector_build(p);
9117         if (status)
9118                 goto error;
9119
9120         status = learner_build(p);
9121         if (status)
9122                 goto error;
9123
9124         status = table_state_build(p);
9125         if (status)
9126                 goto error;
9127
9128         status = regarray_build(p);
9129         if (status)
9130                 goto error;
9131
9132         status = metarray_build(p);
9133         if (status)
9134                 goto error;
9135
9136         p->build_done = 1;
9137
9138         pipeline_compile(p);
9139
9140         return 0;
9141
9142 error:
9143         metarray_build_free(p);
9144         regarray_build_free(p);
9145         table_state_build_free(p);
9146         learner_build_free(p);
9147         selector_build_free(p);
9148         table_build_free(p);
9149         action_build_free(p);
9150         instruction_table_build_free(p);
9151         metadata_build_free(p);
9152         header_build_free(p);
9153         extern_func_build_free(p);
9154         extern_obj_build_free(p);
9155         port_out_build_free(p);
9156         port_in_build_free(p);
9157         struct_build_free(p);
9158
9159         return status;
9160 }
9161
9162 void
9163 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9164 {
9165         uint32_t i;
9166
9167         for (i = 0; i < n_instructions; i++)
9168                 instr_exec(p);
9169 }
9170
9171 void
9172 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9173 {
9174         uint32_t i;
9175
9176         for (i = 0; i < p->n_ports_out; i++) {
9177                 struct port_out_runtime *port = &p->out[i];
9178
9179                 if (port->flush)
9180                         port->flush(port->obj);
9181         }
9182 }
9183
9184 /*
9185  * Control.
9186  */
9187 int
9188 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9189                               struct rte_swx_ctl_pipeline_info *pipeline)
9190 {
9191         struct action *action;
9192         struct table *table;
9193         uint32_t n_actions = 0, n_tables = 0;
9194
9195         if (!p || !pipeline)
9196                 return -EINVAL;
9197
9198         TAILQ_FOREACH(action, &p->actions, node)
9199                 n_actions++;
9200
9201         TAILQ_FOREACH(table, &p->tables, node)
9202                 n_tables++;
9203
9204         pipeline->n_ports_in = p->n_ports_in;
9205         pipeline->n_ports_out = p->n_ports_out;
9206         pipeline->n_actions = n_actions;
9207         pipeline->n_tables = n_tables;
9208         pipeline->n_selectors = p->n_selectors;
9209         pipeline->n_learners = p->n_learners;
9210         pipeline->n_regarrays = p->n_regarrays;
9211         pipeline->n_metarrays = p->n_metarrays;
9212
9213         return 0;
9214 }
9215
9216 int
9217 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9218 {
9219         if (!p || !numa_node)
9220                 return -EINVAL;
9221
9222         *numa_node = p->numa_node;
9223         return 0;
9224 }
9225
9226 int
9227 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9228                             uint32_t action_id,
9229                             struct rte_swx_ctl_action_info *action)
9230 {
9231         struct action *a = NULL;
9232
9233         if (!p || (action_id >= p->n_actions) || !action)
9234                 return -EINVAL;
9235
9236         a = action_find_by_id(p, action_id);
9237         if (!a)
9238                 return -EINVAL;
9239
9240         strcpy(action->name, a->name);
9241         action->n_args = a->st ? a->st->n_fields : 0;
9242         return 0;
9243 }
9244
9245 int
9246 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9247                                 uint32_t action_id,
9248                                 uint32_t action_arg_id,
9249                                 struct rte_swx_ctl_action_arg_info *action_arg)
9250 {
9251         struct action *a = NULL;
9252         struct field *arg = NULL;
9253
9254         if (!p || (action_id >= p->n_actions) || !action_arg)
9255                 return -EINVAL;
9256
9257         a = action_find_by_id(p, action_id);
9258         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9259                 return -EINVAL;
9260
9261         arg = &a->st->fields[action_arg_id];
9262         strcpy(action_arg->name, arg->name);
9263         action_arg->n_bits = arg->n_bits;
9264         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9265
9266         return 0;
9267 }
9268
9269 int
9270 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9271                            uint32_t table_id,
9272                            struct rte_swx_ctl_table_info *table)
9273 {
9274         struct table *t = NULL;
9275
9276         if (!p || !table)
9277                 return -EINVAL;
9278
9279         t = table_find_by_id(p, table_id);
9280         if (!t)
9281                 return -EINVAL;
9282
9283         strcpy(table->name, t->name);
9284         strcpy(table->args, t->args);
9285         table->n_match_fields = t->n_fields;
9286         table->n_actions = t->n_actions;
9287         table->default_action_is_const = t->default_action_is_const;
9288         table->size = t->size;
9289         return 0;
9290 }
9291
9292 int
9293 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9294         uint32_t table_id,
9295         uint32_t match_field_id,
9296         struct rte_swx_ctl_table_match_field_info *match_field)
9297 {
9298         struct table *t;
9299         struct match_field *f;
9300
9301         if (!p || (table_id >= p->n_tables) || !match_field)
9302                 return -EINVAL;
9303
9304         t = table_find_by_id(p, table_id);
9305         if (!t || (match_field_id >= t->n_fields))
9306                 return -EINVAL;
9307
9308         f = &t->fields[match_field_id];
9309         match_field->match_type = f->match_type;
9310         match_field->is_header = t->header ? 1 : 0;
9311         match_field->n_bits = f->field->n_bits;
9312         match_field->offset = f->field->offset;
9313
9314         return 0;
9315 }
9316
9317 int
9318 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9319         uint32_t table_id,
9320         uint32_t table_action_id,
9321         struct rte_swx_ctl_table_action_info *table_action)
9322 {
9323         struct table *t;
9324
9325         if (!p || (table_id >= p->n_tables) || !table_action)
9326                 return -EINVAL;
9327
9328         t = table_find_by_id(p, table_id);
9329         if (!t || (table_action_id >= t->n_actions))
9330                 return -EINVAL;
9331
9332         table_action->action_id = t->actions[table_action_id]->id;
9333
9334         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9335         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9336
9337         return 0;
9338 }
9339
9340 int
9341 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9342                           uint32_t table_id,
9343                           struct rte_swx_table_ops *table_ops,
9344                           int *is_stub)
9345 {
9346         struct table *t;
9347
9348         if (!p || (table_id >= p->n_tables))
9349                 return -EINVAL;
9350
9351         t = table_find_by_id(p, table_id);
9352         if (!t)
9353                 return -EINVAL;
9354
9355         if (t->type) {
9356                 if (table_ops)
9357                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9358                 *is_stub = 0;
9359         } else {
9360                 *is_stub = 1;
9361         }
9362
9363         return 0;
9364 }
9365
9366 int
9367 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9368                               uint32_t selector_id,
9369                               struct rte_swx_ctl_selector_info *selector)
9370 {
9371         struct selector *s = NULL;
9372
9373         if (!p || !selector)
9374                 return -EINVAL;
9375
9376         s = selector_find_by_id(p, selector_id);
9377         if (!s)
9378                 return -EINVAL;
9379
9380         strcpy(selector->name, s->name);
9381
9382         selector->n_selector_fields = s->n_selector_fields;
9383         selector->n_groups_max = s->n_groups_max;
9384         selector->n_members_per_group_max = s->n_members_per_group_max;
9385
9386         return 0;
9387 }
9388
9389 int
9390 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9391          uint32_t selector_id,
9392          struct rte_swx_ctl_table_match_field_info *field)
9393 {
9394         struct selector *s;
9395
9396         if (!p || (selector_id >= p->n_selectors) || !field)
9397                 return -EINVAL;
9398
9399         s = selector_find_by_id(p, selector_id);
9400         if (!s)
9401                 return -EINVAL;
9402
9403         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9404         field->is_header = 0;
9405         field->n_bits = s->group_id_field->n_bits;
9406         field->offset = s->group_id_field->offset;
9407
9408         return 0;
9409 }
9410
9411 int
9412 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9413          uint32_t selector_id,
9414          uint32_t selector_field_id,
9415          struct rte_swx_ctl_table_match_field_info *field)
9416 {
9417         struct selector *s;
9418         struct field *f;
9419
9420         if (!p || (selector_id >= p->n_selectors) || !field)
9421                 return -EINVAL;
9422
9423         s = selector_find_by_id(p, selector_id);
9424         if (!s || (selector_field_id >= s->n_selector_fields))
9425                 return -EINVAL;
9426
9427         f = s->selector_fields[selector_field_id];
9428         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9429         field->is_header = s->selector_header ? 1 : 0;
9430         field->n_bits = f->n_bits;
9431         field->offset = f->offset;
9432
9433         return 0;
9434 }
9435
9436 int
9437 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9438          uint32_t selector_id,
9439          struct rte_swx_ctl_table_match_field_info *field)
9440 {
9441         struct selector *s;
9442
9443         if (!p || (selector_id >= p->n_selectors) || !field)
9444                 return -EINVAL;
9445
9446         s = selector_find_by_id(p, selector_id);
9447         if (!s)
9448                 return -EINVAL;
9449
9450         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9451         field->is_header = 0;
9452         field->n_bits = s->member_id_field->n_bits;
9453         field->offset = s->member_id_field->offset;
9454
9455         return 0;
9456 }
9457
9458 int
9459 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9460                              uint32_t learner_id,
9461                              struct rte_swx_ctl_learner_info *learner)
9462 {
9463         struct learner *l = NULL;
9464
9465         if (!p || !learner)
9466                 return -EINVAL;
9467
9468         l = learner_find_by_id(p, learner_id);
9469         if (!l)
9470                 return -EINVAL;
9471
9472         strcpy(learner->name, l->name);
9473
9474         learner->n_match_fields = l->n_fields;
9475         learner->n_actions = l->n_actions;
9476         learner->default_action_is_const = l->default_action_is_const;
9477         learner->size = l->size;
9478
9479         return 0;
9480 }
9481
9482 int
9483 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9484                                          uint32_t learner_id,
9485                                          uint32_t match_field_id,
9486                                          struct rte_swx_ctl_table_match_field_info *match_field)
9487 {
9488         struct learner *l;
9489         struct field *f;
9490
9491         if (!p || (learner_id >= p->n_learners) || !match_field)
9492                 return -EINVAL;
9493
9494         l = learner_find_by_id(p, learner_id);
9495         if (!l || (match_field_id >= l->n_fields))
9496                 return -EINVAL;
9497
9498         f = l->fields[match_field_id];
9499         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9500         match_field->is_header = l->header ? 1 : 0;
9501         match_field->n_bits = f->n_bits;
9502         match_field->offset = f->offset;
9503
9504         return 0;
9505 }
9506
9507 int
9508 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9509                                     uint32_t learner_id,
9510                                     uint32_t learner_action_id,
9511                                     struct rte_swx_ctl_table_action_info *learner_action)
9512 {
9513         struct learner *l;
9514
9515         if (!p || (learner_id >= p->n_learners) || !learner_action)
9516                 return -EINVAL;
9517
9518         l = learner_find_by_id(p, learner_id);
9519         if (!l || (learner_action_id >= l->n_actions))
9520                 return -EINVAL;
9521
9522         learner_action->action_id = l->actions[learner_action_id]->id;
9523
9524         learner_action->action_is_for_table_entries =
9525                 l->action_is_for_table_entries[learner_action_id];
9526
9527         learner_action->action_is_for_default_entry =
9528                 l->action_is_for_default_entry[learner_action_id];
9529
9530         return 0;
9531 }
9532
9533 int
9534 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9535                                  struct rte_swx_table_state **table_state)
9536 {
9537         if (!p || !table_state || !p->build_done)
9538                 return -EINVAL;
9539
9540         *table_state = p->table_state;
9541         return 0;
9542 }
9543
9544 int
9545 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9546                                  struct rte_swx_table_state *table_state)
9547 {
9548         if (!p || !table_state || !p->build_done)
9549                 return -EINVAL;
9550
9551         p->table_state = table_state;
9552         return 0;
9553 }
9554
9555 int
9556 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9557                                         uint32_t port_id,
9558                                         struct rte_swx_port_in_stats *stats)
9559 {
9560         struct port_in *port;
9561
9562         if (!p || !stats)
9563                 return -EINVAL;
9564
9565         port = port_in_find(p, port_id);
9566         if (!port)
9567                 return -EINVAL;
9568
9569         port->type->ops.stats_read(port->obj, stats);
9570         return 0;
9571 }
9572
9573 int
9574 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9575                                          uint32_t port_id,
9576                                          struct rte_swx_port_out_stats *stats)
9577 {
9578         struct port_out *port;
9579
9580         if (!p || !stats)
9581                 return -EINVAL;
9582
9583         port = port_out_find(p, port_id);
9584         if (!port)
9585                 return -EINVAL;
9586
9587         port->type->ops.stats_read(port->obj, stats);
9588         return 0;
9589 }
9590
9591 int
9592 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9593                                       const char *table_name,
9594                                       struct rte_swx_table_stats *stats)
9595 {
9596         struct table *table;
9597         struct table_statistics *table_stats;
9598
9599         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9600                 return -EINVAL;
9601
9602         table = table_find(p, table_name);
9603         if (!table)
9604                 return -EINVAL;
9605
9606         table_stats = &p->table_stats[table->id];
9607
9608         memcpy(stats->n_pkts_action,
9609                table_stats->n_pkts_action,
9610                p->n_actions * sizeof(uint64_t));
9611
9612         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9613         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9614
9615         return 0;
9616 }
9617
9618 int
9619 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9620         const char *selector_name,
9621         struct rte_swx_pipeline_selector_stats *stats)
9622 {
9623         struct selector *s;
9624
9625         if (!p || !selector_name || !selector_name[0] || !stats)
9626                 return -EINVAL;
9627
9628         s = selector_find(p, selector_name);
9629         if (!s)
9630                 return -EINVAL;
9631
9632         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9633
9634         return 0;
9635 }
9636
9637 int
9638 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9639                                         const char *learner_name,
9640                                         struct rte_swx_learner_stats *stats)
9641 {
9642         struct learner *l;
9643         struct learner_statistics *learner_stats;
9644
9645         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9646                 return -EINVAL;
9647
9648         l = learner_find(p, learner_name);
9649         if (!l)
9650                 return -EINVAL;
9651
9652         learner_stats = &p->learner_stats[l->id];
9653
9654         memcpy(stats->n_pkts_action,
9655                learner_stats->n_pkts_action,
9656                p->n_actions * sizeof(uint64_t));
9657
9658         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9659         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9660
9661         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9662         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9663
9664         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9665
9666         return 0;
9667 }
9668
9669 int
9670 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9671                               uint32_t regarray_id,
9672                               struct rte_swx_ctl_regarray_info *regarray)
9673 {
9674         struct regarray *r;
9675
9676         if (!p || !regarray)
9677                 return -EINVAL;
9678
9679         r = regarray_find_by_id(p, regarray_id);
9680         if (!r)
9681                 return -EINVAL;
9682
9683         strcpy(regarray->name, r->name);
9684         regarray->size = r->size;
9685         return 0;
9686 }
9687
9688 int
9689 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9690                                    const char *regarray_name,
9691                                    uint32_t regarray_index,
9692                                    uint64_t *value)
9693 {
9694         struct regarray *regarray;
9695         struct regarray_runtime *r;
9696
9697         if (!p || !regarray_name || !value)
9698                 return -EINVAL;
9699
9700         regarray = regarray_find(p, regarray_name);
9701         if (!regarray || (regarray_index >= regarray->size))
9702                 return -EINVAL;
9703
9704         r = &p->regarray_runtime[regarray->id];
9705         *value = r->regarray[regarray_index];
9706         return 0;
9707 }
9708
9709 int
9710 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9711                                    const char *regarray_name,
9712                                    uint32_t regarray_index,
9713                                    uint64_t value)
9714 {
9715         struct regarray *regarray;
9716         struct regarray_runtime *r;
9717
9718         if (!p || !regarray_name)
9719                 return -EINVAL;
9720
9721         regarray = regarray_find(p, regarray_name);
9722         if (!regarray || (regarray_index >= regarray->size))
9723                 return -EINVAL;
9724
9725         r = &p->regarray_runtime[regarray->id];
9726         r->regarray[regarray_index] = value;
9727         return 0;
9728 }
9729
9730 int
9731 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9732                               uint32_t metarray_id,
9733                               struct rte_swx_ctl_metarray_info *metarray)
9734 {
9735         struct metarray *m;
9736
9737         if (!p || !metarray)
9738                 return -EINVAL;
9739
9740         m = metarray_find_by_id(p, metarray_id);
9741         if (!m)
9742                 return -EINVAL;
9743
9744         strcpy(metarray->name, m->name);
9745         metarray->size = m->size;
9746         return 0;
9747 }
9748
9749 int
9750 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9751                               const char *name,
9752                               struct rte_meter_trtcm_params *params)
9753 {
9754         struct meter_profile *mp;
9755         int status;
9756
9757         CHECK(p, EINVAL);
9758         CHECK_NAME(name, EINVAL);
9759         CHECK(params, EINVAL);
9760         CHECK(!meter_profile_find(p, name), EEXIST);
9761
9762         /* Node allocation. */
9763         mp = calloc(1, sizeof(struct meter_profile));
9764         CHECK(mp, ENOMEM);
9765
9766         /* Node initialization. */
9767         strcpy(mp->name, name);
9768         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9769         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9770         if (status) {
9771                 free(mp);
9772                 CHECK(0, EINVAL);
9773         }
9774
9775         /* Node add to tailq. */
9776         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9777
9778         return 0;
9779 }
9780
9781 int
9782 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9783                                  const char *name)
9784 {
9785         struct meter_profile *mp;
9786
9787         CHECK(p, EINVAL);
9788         CHECK_NAME(name, EINVAL);
9789
9790         mp = meter_profile_find(p, name);
9791         CHECK(mp, EINVAL);
9792         CHECK(!mp->n_users, EBUSY);
9793
9794         /* Remove node from tailq. */
9795         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9796         free(mp);
9797
9798         return 0;
9799 }
9800
9801 int
9802 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9803                         const char *metarray_name,
9804                         uint32_t metarray_index)
9805 {
9806         struct meter_profile *mp_old;
9807         struct metarray *metarray;
9808         struct metarray_runtime *metarray_runtime;
9809         struct meter *m;
9810
9811         CHECK(p, EINVAL);
9812         CHECK_NAME(metarray_name, EINVAL);
9813
9814         metarray = metarray_find(p, metarray_name);
9815         CHECK(metarray, EINVAL);
9816         CHECK(metarray_index < metarray->size, EINVAL);
9817
9818         metarray_runtime = &p->metarray_runtime[metarray->id];
9819         m = &metarray_runtime->metarray[metarray_index];
9820         mp_old = m->profile;
9821
9822         meter_init(m);
9823
9824         mp_old->n_users--;
9825
9826         return 0;
9827 }
9828
9829 int
9830 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9831                       const char *metarray_name,
9832                       uint32_t metarray_index,
9833                       const char *profile_name)
9834 {
9835         struct meter_profile *mp, *mp_old;
9836         struct metarray *metarray;
9837         struct metarray_runtime *metarray_runtime;
9838         struct meter *m;
9839
9840         CHECK(p, EINVAL);
9841         CHECK_NAME(metarray_name, EINVAL);
9842
9843         metarray = metarray_find(p, metarray_name);
9844         CHECK(metarray, EINVAL);
9845         CHECK(metarray_index < metarray->size, EINVAL);
9846
9847         mp = meter_profile_find(p, profile_name);
9848         CHECK(mp, EINVAL);
9849
9850         metarray_runtime = &p->metarray_runtime[metarray->id];
9851         m = &metarray_runtime->metarray[metarray_index];
9852         mp_old = m->profile;
9853
9854         memset(m, 0, sizeof(struct meter));
9855         rte_meter_trtcm_config(&m->m, &mp->profile);
9856         m->profile = mp;
9857         m->color_mask = RTE_COLORS;
9858
9859         mp->n_users++;
9860         mp_old->n_users--;
9861
9862         return 0;
9863 }
9864
9865 int
9866 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9867                              const char *metarray_name,
9868                              uint32_t metarray_index,
9869                              struct rte_swx_ctl_meter_stats *stats)
9870 {
9871         struct metarray *metarray;
9872         struct metarray_runtime *metarray_runtime;
9873         struct meter *m;
9874
9875         CHECK(p, EINVAL);
9876         CHECK_NAME(metarray_name, EINVAL);
9877
9878         metarray = metarray_find(p, metarray_name);
9879         CHECK(metarray, EINVAL);
9880         CHECK(metarray_index < metarray->size, EINVAL);
9881
9882         CHECK(stats, EINVAL);
9883
9884         metarray_runtime = &p->metarray_runtime[metarray->id];
9885         m = &metarray_runtime->metarray[metarray_index];
9886
9887         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9888         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9889
9890         return 0;
9891 }
9892
9893 /*
9894  * Pipeline compilation.
9895  */
9896 static const char *
9897 instr_type_to_name(struct instruction *instr)
9898 {
9899         switch (instr->type) {
9900         case INSTR_RX: return "INSTR_RX";
9901
9902         case INSTR_TX: return "INSTR_TX";
9903         case INSTR_TX_I: return "INSTR_TX_I";
9904
9905         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
9906         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
9907         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
9908         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
9909         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
9910         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
9911         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
9912         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
9913
9914         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
9915
9916         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
9917
9918         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
9919         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
9920         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
9921         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
9922         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
9923         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
9924         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
9925         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
9926         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
9927
9928         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
9929         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
9930
9931         case INSTR_MOV: return "INSTR_MOV";
9932         case INSTR_MOV_MH: return "INSTR_MOV_MH";
9933         case INSTR_MOV_HM: return "INSTR_MOV_HM";
9934         case INSTR_MOV_HH: return "INSTR_MOV_HH";
9935         case INSTR_MOV_I: return "INSTR_MOV_I";
9936
9937         case INSTR_DMA_HT: return "INSTR_DMA_HT";
9938         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
9939         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
9940         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
9941         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
9942         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
9943         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
9944         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
9945
9946         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
9947         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
9948         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
9949         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
9950         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
9951         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
9952
9953         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
9954         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
9955         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
9956         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
9957         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
9958         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
9959
9960         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
9961         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
9962         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
9963         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
9964
9965         case INSTR_ALU_AND: return "INSTR_ALU_AND";
9966         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
9967         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
9968         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
9969         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
9970
9971         case INSTR_ALU_OR: return "INSTR_ALU_OR";
9972         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
9973         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
9974         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
9975         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
9976
9977         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
9978         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
9979         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
9980         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
9981         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
9982
9983         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
9984         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
9985         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
9986         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
9987         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
9988         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
9989
9990         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
9991         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
9992         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
9993         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
9994         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
9995         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
9996
9997         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
9998         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
9999         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10000
10001         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10002         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10003         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10004         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10005         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10006         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10007
10008         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10009         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10010         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10011         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10012         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10013         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10014         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10015         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10016         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10017
10018         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10019         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10020         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10021         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10022         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10023         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10024         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10025         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10026         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10027
10028         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10029         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10030         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10031
10032         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10033         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10034         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10035         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10036         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10037         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10038         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10039         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10040         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10041         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10042         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10043         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10044
10045         case INSTR_TABLE: return "INSTR_TABLE";
10046         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10047         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10048         case INSTR_LEARNER: return "INSTR_LEARNER";
10049         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10050
10051         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10052         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10053
10054         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10055         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10056
10057         case INSTR_JMP: return "INSTR_JMP";
10058         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10059         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10060         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10061         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10062         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10063         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10064         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10065         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10066         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10067         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10068         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10069         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10070         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10071         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10072         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10073         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10074         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10075         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10076         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10077         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10078         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10079         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10080         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10081         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10082         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10083         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10084         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10085         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10086
10087         case INSTR_RETURN: return "INSTR_RETURN";
10088
10089         default: return "INSTR_UNKNOWN";
10090         }
10091 }
10092
10093 typedef void
10094 (*instruction_export_t)(struct instruction *, FILE *);
10095
10096 static void
10097 instr_io_export(struct instruction *instr, FILE *f)
10098 {
10099         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10100
10101         /* n_io, n_io_imm, n_hdrs. */
10102         if (instr->type == INSTR_RX ||
10103             instr->type == INSTR_TX ||
10104             instr->type == INSTR_HDR_EXTRACT_M ||
10105             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10106                 n_io = 1;
10107
10108         if (instr->type == INSTR_TX_I)
10109                 n_io_imm = 1;
10110
10111         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10112                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10113
10114         if (instr->type == INSTR_HDR_EXTRACT_M ||
10115             instr->type == INSTR_HDR_LOOKAHEAD ||
10116             instr->type == INSTR_HDR_EMIT)
10117                 n_hdrs = 1;
10118
10119         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10120                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10121
10122         /* instr. */
10123         fprintf(f,
10124                 "\t{\n"
10125                 "\t\t.type = %s,\n",
10126                 instr_type_to_name(instr));
10127
10128         /* instr.io. */
10129         fprintf(f,
10130                 "\t\t.io = {\n");
10131
10132         /* instr.io.io. */
10133         if (n_io)
10134                 fprintf(f,
10135                         "\t\t\t.io = {\n"
10136                         "\t\t\t\t.offset = %u,\n"
10137                         "\t\t\t\t.n_bits = %u,\n"
10138                         "\t\t\t},\n",
10139                         instr->io.io.offset,
10140                         instr->io.io.n_bits);
10141
10142         if (n_io_imm)
10143                 fprintf(f,
10144                         "\t\t\t.io = {\n"
10145                         "\t\t\t\t.val = %u,\n"
10146                         "\t\t\t},\n",
10147                         instr->io.io.val);
10148
10149         /* instr.io.hdr. */
10150         if (n_hdrs) {
10151                 fprintf(f,
10152                         "\t\t.hdr = {\n");
10153
10154                 /* instr.io.hdr.header_id. */
10155                 fprintf(f,
10156                         "\t\t\t.header_id = {");
10157
10158                 for (i = 0; i < n_hdrs; i++)
10159                         fprintf(f,
10160                                 "%u, ",
10161                                 instr->io.hdr.header_id[i]);
10162
10163                 fprintf(f,
10164                         "},\n");
10165
10166                 /* instr.io.hdr.struct_id. */
10167                 fprintf(f,
10168                         "\t\t\t.struct_id = {");
10169
10170                 for (i = 0; i < n_hdrs; i++)
10171                         fprintf(f,
10172                                 "%u, ",
10173                                 instr->io.hdr.struct_id[i]);
10174
10175                 fprintf(f,
10176                         "},\n");
10177
10178                 /* instr.io.hdr.n_bytes. */
10179                 fprintf(f,
10180                         "\t\t\t.n_bytes = {");
10181
10182                 for (i = 0; i < n_hdrs; i++)
10183                         fprintf(f,
10184                                 "%u, ",
10185                                 instr->io.hdr.n_bytes[i]);
10186
10187                 fprintf(f,
10188                         "},\n");
10189
10190                 /* instr.io.hdr - closing curly brace. */
10191                 fprintf(f,
10192                         "\t\t\t}\n,");
10193         }
10194
10195         /* instr.io - closing curly brace. */
10196         fprintf(f,
10197                 "\t\t},\n");
10198
10199         /* instr - closing curly brace. */
10200         fprintf(f,
10201                 "\t},\n");
10202 }
10203
10204 static void
10205 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10206 {
10207         fprintf(f,
10208                 "\t{\n"
10209                 "\t\t.type = %s,\n"
10210                 "\t\t.valid = {\n"
10211                 "\t\t\t.header_id = %u,\n"
10212                 "\t\t},\n"
10213                 "\t},\n",
10214                 instr_type_to_name(instr),
10215                 instr->valid.header_id);
10216 }
10217
10218 static void
10219 instr_mov_export(struct instruction *instr, FILE *f)
10220 {
10221         if (instr->type != INSTR_MOV_I)
10222                 fprintf(f,
10223                         "\t{\n"
10224                         "\t\t.type = %s,\n"
10225                         "\t\t.mov = {\n"
10226                         "\t\t\t.dst = {\n"
10227                         "\t\t\t\t.struct_id = %u,\n"
10228                         "\t\t\t\t.n_bits = %u,\n"
10229                         "\t\t\t\t.offset = %u,\n"
10230                         "\t\t\t},\n"
10231                         "\t\t\t.src = {\n"
10232                         "\t\t\t\t.struct_id = %u,\n"
10233                         "\t\t\t\t.n_bits = %u,\n"
10234                         "\t\t\t\t.offset = %u,\n"
10235                         "\t\t\t},\n"
10236                         "\t\t},\n"
10237                         "\t},\n",
10238                         instr_type_to_name(instr),
10239                         instr->mov.dst.struct_id,
10240                         instr->mov.dst.n_bits,
10241                         instr->mov.dst.offset,
10242                         instr->mov.src.struct_id,
10243                         instr->mov.src.n_bits,
10244                         instr->mov.src.offset);
10245         else
10246                 fprintf(f,
10247                         "\t{\n"
10248                         "\t\t.type = %s,\n"
10249                         "\t\t.mov = {\n"
10250                         "\t\t\t.dst = {\n"
10251                         "\t\t\t\t.struct_id = %u,\n"
10252                         "\t\t\t\t.n_bits = %u,\n"
10253                         "\t\t\t\t.offset = %u,\n"
10254                         "\t\t\t}\n,"
10255                         "\t\t\t.src_val = %" PRIu64 ",\n"
10256                         "\t\t},\n"
10257                         "\t},\n",
10258                         instr_type_to_name(instr),
10259                         instr->mov.dst.struct_id,
10260                         instr->mov.dst.n_bits,
10261                         instr->mov.dst.offset,
10262                         instr->mov.src_val);
10263 }
10264
10265 static void
10266 instr_dma_ht_export(struct instruction *instr, FILE *f)
10267 {
10268         uint32_t n_dma = 0, i;
10269
10270         /* n_dma. */
10271         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10272
10273         /* instr. */
10274         fprintf(f,
10275                 "\t{\n"
10276                 "\t\t.type = %s,\n",
10277                 instr_type_to_name(instr));
10278
10279         /* instr.dma. */
10280         fprintf(f,
10281                 "\t\t.dma = {\n");
10282
10283         /* instr.dma.dst. */
10284         fprintf(f,
10285                 "\t\t\t.dst = {\n");
10286
10287         /* instr.dma.dst.header_id. */
10288         fprintf(f,
10289                 "\t\t\t\t.header_id = {");
10290
10291         for (i = 0; i < n_dma; i++)
10292                 fprintf(f,
10293                         "%u, ",
10294                         instr->dma.dst.header_id[i]);
10295
10296         fprintf(f,
10297                 "},\n");
10298
10299         /* instr.dma.dst.struct_id. */
10300         fprintf(f,
10301                 "\t\t\t\t.struct_id = {");
10302
10303         for (i = 0; i < n_dma; i++)
10304                 fprintf(f,
10305                         "%u, ",
10306                         instr->dma.dst.struct_id[i]);
10307
10308         fprintf(f,
10309                 "},\n");
10310
10311         /* instr.dma.dst - closing curly brace. */
10312         fprintf(f,
10313                 "\t\t\t},\n");
10314
10315         /* instr.dma.src. */
10316         fprintf(f,
10317                 "\t\t\t.src = {\n");
10318
10319         /* instr.dma.src.offset. */
10320         fprintf(f,
10321                 "\t\t\t\t.offset = {");
10322
10323         for (i = 0; i < n_dma; i++)
10324                 fprintf(f,
10325                         "%u, ",
10326                         instr->dma.src.offset[i]);
10327
10328         fprintf(f,
10329                 "},\n");
10330
10331         /* instr.dma.src - closing curly brace. */
10332         fprintf(f,
10333                 "\t\t\t},\n");
10334
10335         /* instr.dma.n_bytes. */
10336         fprintf(f,
10337                 "\t\t\t.n_bytes = {");
10338
10339         for (i = 0; i < n_dma; i++)
10340                 fprintf(f,
10341                         "%u, ",
10342                         instr->dma.n_bytes[i]);
10343
10344         fprintf(f,
10345                 "},\n");
10346
10347         /* instr.dma - closing curly brace. */
10348         fprintf(f,
10349                 "\t\t},\n");
10350
10351         /* instr - closing curly brace. */
10352         fprintf(f,
10353                 "\t},\n");
10354 }
10355
10356 static void
10357 instr_alu_export(struct instruction *instr, FILE *f)
10358 {
10359         int imm = 0;
10360
10361         if (instr->type == INSTR_ALU_ADD_MI ||
10362             instr->type == INSTR_ALU_ADD_HI ||
10363             instr->type == INSTR_ALU_SUB_MI ||
10364             instr->type == INSTR_ALU_SUB_HI ||
10365             instr->type == INSTR_ALU_SHL_MI ||
10366             instr->type == INSTR_ALU_SHL_HI ||
10367             instr->type == INSTR_ALU_SHR_MI ||
10368             instr->type == INSTR_ALU_SHR_HI ||
10369             instr->type == INSTR_ALU_AND_I ||
10370             instr->type == INSTR_ALU_OR_I ||
10371             instr->type == INSTR_ALU_XOR_I)
10372                 imm = 1;
10373
10374         if (!imm)
10375                 fprintf(f,
10376                         "\t{\n"
10377                         "\t\t.type = %s,\n"
10378                         "\t\t.alu = {\n"
10379                         "\t\t\t.dst = {\n"
10380                         "\t\t\t\t.struct_id = %u,\n"
10381                         "\t\t\t\t.n_bits = %u,\n"
10382                         "\t\t\t\t.offset = %u,\n"
10383                         "\t\t\t},\n"
10384                         "\t\t\t.src = {\n"
10385                         "\t\t\t\t.struct_id = %u,\n"
10386                         "\t\t\t\t.n_bits = %u,\n"
10387                         "\t\t\t\t.offset = %u,\n"
10388                         "\t\t\t},\n"
10389                         "\t\t},\n"
10390                         "\t},\n",
10391                         instr_type_to_name(instr),
10392                         instr->alu.dst.struct_id,
10393                         instr->alu.dst.n_bits,
10394                         instr->alu.dst.offset,
10395                         instr->alu.src.struct_id,
10396                         instr->alu.src.n_bits,
10397                         instr->alu.src.offset);
10398         else
10399                 fprintf(f,
10400                         "\t{\n"
10401                         "\t\t.type = %s,\n"
10402                         "\t\t.alu = {\n"
10403                         "\t\t\t.dst = {\n"
10404                         "\t\t\t\t.struct_id = %u,\n"
10405                         "\t\t\t\t.n_bits = %u,\n"
10406                         "\t\t\t\t.offset = %u,\n"
10407                         "\t\t\t}\n,"
10408                         "\t\t\t.src_val = %" PRIu64 ",\n"
10409                         "\t\t},\n"
10410                         "\t},\n",
10411                         instr_type_to_name(instr),
10412                         instr->alu.dst.struct_id,
10413                         instr->alu.dst.n_bits,
10414                         instr->alu.dst.offset,
10415                         instr->alu.src_val);
10416 }
10417
10418 static void
10419 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10420 {
10421         int prefetch  = 0, idx_imm = 0, src_imm = 0;
10422
10423         if (instr->type == INSTR_REGPREFETCH_RH ||
10424             instr->type == INSTR_REGPREFETCH_RM ||
10425             instr->type == INSTR_REGPREFETCH_RI)
10426                 prefetch = 1;
10427
10428         /* index is the 3rd operand for the regrd instruction and the 2nd
10429          * operand for the regwr and regadd instructions.
10430          */
10431         if (instr->type == INSTR_REGPREFETCH_RI ||
10432             instr->type == INSTR_REGRD_HRI ||
10433             instr->type == INSTR_REGRD_MRI ||
10434             instr->type == INSTR_REGWR_RIH ||
10435             instr->type == INSTR_REGWR_RIM ||
10436             instr->type == INSTR_REGWR_RII ||
10437             instr->type == INSTR_REGADD_RIH ||
10438             instr->type == INSTR_REGADD_RIM ||
10439             instr->type == INSTR_REGADD_RII)
10440                 idx_imm = 1;
10441
10442         /* src is the 3rd operand for the regwr and regadd instructions. */
10443         if (instr->type == INSTR_REGWR_RHI ||
10444             instr->type == INSTR_REGWR_RMI ||
10445             instr->type == INSTR_REGWR_RII ||
10446             instr->type == INSTR_REGADD_RHI ||
10447             instr->type == INSTR_REGADD_RMI ||
10448             instr->type == INSTR_REGADD_RII)
10449                 src_imm = 1;
10450
10451         /* instr.regarray.regarray_id. */
10452         fprintf(f,
10453                 "\t{\n"
10454                 "\t\t.type = %s,\n"
10455                 "\t\t.regarray = {\n"
10456                 "\t\t\t.regarray_id = %u,\n",
10457                 instr_type_to_name(instr),
10458                 instr->regarray.regarray_id);
10459
10460         /* instr.regarray.idx / instr.regarray.idx_val. */
10461         if (!idx_imm)
10462                 fprintf(f,
10463                         "\t\t\t\t.idx = {\n"
10464                         "\t\t\t\t\t.struct_id = %u,\n"
10465                         "\t\t\t\t\t.n_bits = %u,\n"
10466                         "\t\t\t\t\t.offset = %u,\n"
10467                         "\t\t\t\t},\n",
10468                         instr->regarray.idx.struct_id,
10469                         instr->regarray.idx.n_bits,
10470                         instr->regarray.idx.offset);
10471         else
10472                 fprintf(f,
10473                         "\t\t\t\t.idx_val = %u,\n",
10474                         instr->regarray.idx_val);
10475
10476         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10477         if (!prefetch) {
10478                 if (!src_imm)
10479                         fprintf(f,
10480                                 "\t\t\t\t.dstsrc = {\n"
10481                                 "\t\t\t\t\t.struct_id = %u,\n"
10482                                 "\t\t\t\t\t.n_bits = %u,\n"
10483                                 "\t\t\t\t\t.offset = %u,\n"
10484                                 "\t\t\t\t},\n",
10485                                 instr->regarray.dstsrc.struct_id,
10486                                 instr->regarray.dstsrc.n_bits,
10487                                 instr->regarray.dstsrc.offset);
10488                 else
10489                         fprintf(f,
10490                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10491                                 instr->regarray.dstsrc_val);
10492         }
10493
10494         /* instr.regarray and instr - closing curly braces. */
10495         fprintf(f,
10496                 "\t\t},\n"
10497                 "\t},\n");
10498 }
10499
10500 static void
10501 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10502 {
10503         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
10504
10505         if (instr->type == INSTR_METPREFETCH_H ||
10506             instr->type == INSTR_METPREFETCH_M ||
10507             instr->type == INSTR_METPREFETCH_I)
10508                 prefetch = 1;
10509
10510         /* idx_imm. */
10511         if (instr->type == INSTR_METPREFETCH_I ||
10512             instr->type == INSTR_METER_IHM ||
10513             instr->type == INSTR_METER_IHI ||
10514             instr->type == INSTR_METER_IMM ||
10515             instr->type == INSTR_METER_IMI)
10516                 idx_imm = 1;
10517
10518         /* color_in_imm. */
10519         if (instr->type == INSTR_METER_HHI ||
10520             instr->type == INSTR_METER_HMI ||
10521             instr->type == INSTR_METER_MHI ||
10522             instr->type == INSTR_METER_MMI ||
10523             instr->type == INSTR_METER_IHI ||
10524             instr->type == INSTR_METER_IMI)
10525                 color_in_imm = 1;
10526
10527         /* instr.meter.metarray_id. */
10528         fprintf(f,
10529                 "\t{\n"
10530                 "\t\t.type = %s,\n"
10531                 "\t\t.meter = {\n"
10532                 "\t\t\t.metarray_id = %u,\n",
10533                 instr_type_to_name(instr),
10534                 instr->meter.metarray_id);
10535
10536         /* instr.meter.idx / instr.meter.idx_val. */
10537         if (!idx_imm)
10538                 fprintf(f,
10539                         "\t\t\t.idx = {\n"
10540                         "\t\t\t\t.struct_id = %u,\n"
10541                         "\t\t\t\t.n_bits = %u,\n"
10542                         "\t\t\t\t.offset = %u,\n"
10543                         "\t\t\t},\n",
10544                         instr->meter.idx.struct_id,
10545                         instr->meter.idx.n_bits,
10546                         instr->meter.idx.offset);
10547         else
10548                 fprintf(f,
10549                         "\t\t\t.idx_val = %u,\n",
10550                         instr->meter.idx_val);
10551
10552         if (!prefetch) {
10553                 /* instr.meter.length. */
10554                 fprintf(f,
10555                         "\t\t\t.length = {\n"
10556                         "\t\t\t\t.struct_id = %u,\n"
10557                         "\t\t\t\t.n_bits = %u,\n"
10558                         "\t\t\t\t.offset = %u,\n"
10559                         "\t\t\t},\n",
10560                         instr->meter.length.struct_id,
10561                         instr->meter.length.n_bits,
10562                         instr->meter.length.offset);
10563
10564                 /* instr.meter.color_in / instr.meter.color_in_val. */
10565                 if (!color_in_imm)
10566                         fprintf(f,
10567                                 "\t\t\t.color_in = {\n"
10568                                 "\t\t\t\t.struct_id = %u,\n"
10569                                 "\t\t\t\t.n_bits = %u,\n"
10570                                 "\t\t\t\t.offset = %u,\n"
10571                                 "\t\t\t},\n",
10572                                 instr->meter.color_in.struct_id,
10573                                 instr->meter.color_in.n_bits,
10574                                 instr->meter.color_in.offset);
10575                 else
10576                         fprintf(f,
10577                                 "\t\t\t.color_in_val = %u,\n",
10578                                 (uint32_t)instr->meter.color_in_val);
10579
10580                 /* instr.meter.color_out. */
10581                 fprintf(f,
10582                         "\t\t\t.color_out = {\n"
10583                         "\t\t\t\t.struct_id = %u,\n"
10584                         "\t\t\t\t.n_bits = %u,\n"
10585                         "\t\t\t\t.offset = %u,\n"
10586                         "\t\t\t},\n",
10587                         instr->meter.color_out.struct_id,
10588                         instr->meter.color_out.n_bits,
10589                         instr->meter.color_out.offset);
10590         }
10591
10592         /* instr.meter and instr - closing curly braces. */
10593         fprintf(f,
10594                 "\t\t},\n"
10595                 "\t},\n");
10596 }
10597
10598 static void
10599 instr_table_export(struct instruction *instr,
10600                 FILE *f)
10601 {
10602         fprintf(f,
10603                 "\t{\n"
10604                 "\t\t.type = %s,\n"
10605                 "\t\t.table = {\n"
10606                 "\t\t\t.table_id = %u,\n"
10607                 "\t\t},\n"
10608                 "\t},\n",
10609                 instr_type_to_name(instr),
10610                 instr->table.table_id);
10611 }
10612
10613 static void
10614 instr_learn_export(struct instruction *instr, FILE *f)
10615 {
10616         fprintf(f,
10617                 "\t{\n"
10618                 "\t\t.type = %s,\n"
10619                 "\t\t.learn = {\n"
10620                 "\t\t\t\t.action_id = %u,\n"
10621                 "\t\t},\n"
10622                 "\t},\n",
10623                 instr_type_to_name(instr),
10624                 instr->learn.action_id);
10625 }
10626
10627 static void
10628 instr_forget_export(struct instruction *instr, FILE *f)
10629 {
10630         fprintf(f,
10631                 "\t{\n"
10632                 "\t\t.type = %s,\n"
10633                 "\t},\n",
10634                 instr_type_to_name(instr));
10635 }
10636
10637 static void
10638 instr_extern_export(struct instruction *instr, FILE *f)
10639 {
10640         if (instr->type == INSTR_EXTERN_OBJ)
10641                 fprintf(f,
10642                         "\t{\n"
10643                         "\t\t.type = %s,\n"
10644                         "\t\t.ext_obj = {\n"
10645                         "\t\t\t.ext_obj_id = %u,\n"
10646                         "\t\t\t.func_id = %u,\n"
10647                         "\t\t},\n"
10648                         "\t},\n",
10649                         instr_type_to_name(instr),
10650                         instr->ext_obj.ext_obj_id,
10651                         instr->ext_obj.func_id);
10652         else
10653                 fprintf(f,
10654                         "\t{\n"
10655                         "\t\t.type = %s,\n"
10656                         "\t\t.ext_func = {\n"
10657                         "\t\t\t.ext_func_id = %u,\n"
10658                         "\t\t},\n"
10659                         "\t},\n",
10660                         instr_type_to_name(instr),
10661                         instr->ext_func.ext_func_id);
10662 }
10663
10664 static void
10665 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10666 {
10667         fprintf(f,
10668                 "\t{\n"
10669                 "\t\t.type = %s,\n"
10670                 "\t\t.jmp = {\n"
10671                 "\t\t\t.ip = NULL,\n",
10672                 instr_type_to_name(instr));
10673
10674         switch (instr->type) {
10675         case INSTR_JMP_VALID:
10676         case INSTR_JMP_INVALID:
10677                 fprintf(f,
10678                         "\t\t\t.header_id = %u,\n",
10679                         instr->jmp.header_id);
10680                 break;
10681
10682         case INSTR_JMP_ACTION_HIT:
10683         case INSTR_JMP_ACTION_MISS:
10684                 fprintf(f,
10685                         "\t\t\t.action_id = %u,\n",
10686                         instr->jmp.action_id);
10687                 break;
10688
10689         case INSTR_JMP_EQ:
10690         case INSTR_JMP_EQ_MH:
10691         case INSTR_JMP_EQ_HM:
10692         case INSTR_JMP_EQ_HH:
10693         case INSTR_JMP_NEQ:
10694         case INSTR_JMP_NEQ_MH:
10695         case INSTR_JMP_NEQ_HM:
10696         case INSTR_JMP_NEQ_HH:
10697         case INSTR_JMP_LT:
10698         case INSTR_JMP_LT_MH:
10699         case INSTR_JMP_LT_HM:
10700         case INSTR_JMP_LT_HH:
10701         case INSTR_JMP_GT:
10702         case INSTR_JMP_GT_MH:
10703         case INSTR_JMP_GT_HM:
10704         case INSTR_JMP_GT_HH:
10705                 fprintf(f,
10706                         "\t\t\t.a = {\n"
10707                         "\t\t\t\t.struct_id = %u,\n"
10708                         "\t\t\t\t.n_bits = %u,\n"
10709                         "\t\t\t\t.offset = %u,\n"
10710                         "\t\t\t},\n"
10711                         "\t\t\t.b = {\n"
10712                         "\t\t\t\t.struct_id = %u,\n"
10713                         "\t\t\t\t.n_bits = %u,\n"
10714                         "\t\t\t\t.offset = %u,\n"
10715                         "\t\t\t},\n",
10716                         instr->jmp.a.struct_id,
10717                         instr->jmp.a.n_bits,
10718                         instr->jmp.a.offset,
10719                         instr->jmp.b.struct_id,
10720                         instr->jmp.b.n_bits,
10721                         instr->jmp.b.offset);
10722                 break;
10723
10724         case INSTR_JMP_EQ_I:
10725         case INSTR_JMP_NEQ_I:
10726         case INSTR_JMP_LT_MI:
10727         case INSTR_JMP_LT_HI:
10728         case INSTR_JMP_GT_MI:
10729         case INSTR_JMP_GT_HI:
10730                 fprintf(f,
10731                         "\t\t\t.a = {\n"
10732                         "\t\t\t\t.struct_id = %u,\n"
10733                         "\t\t\t\t.n_bits = %u,\n"
10734                         "\t\t\t\t.offset = %u,\n"
10735                         "\t\t\t}\n,"
10736                         "\t\t\t.b_val = %" PRIu64 ",\n",
10737                         instr->jmp.a.struct_id,
10738                         instr->jmp.a.n_bits,
10739                         instr->jmp.a.offset,
10740                         instr->jmp.b_val);
10741                 break;
10742
10743         default:
10744                 break;
10745         }
10746
10747         fprintf(f,
10748                 "\t\t},\n"
10749                 "\t},\n");
10750 }
10751
10752 static void
10753 instr_return_export(struct instruction *instr,
10754                 FILE *f)
10755 {
10756         fprintf(f,
10757                 "\t{\n"
10758                 "\t\t.type = %s,\n",
10759                 instr_type_to_name(instr));
10760
10761         fprintf(f,
10762                 "\t},\n");
10763 }
10764
10765 static instruction_export_t export_table[] = {
10766         [INSTR_RX] = instr_io_export,
10767
10768         [INSTR_TX] = instr_io_export,
10769         [INSTR_TX_I] = instr_io_export,
10770
10771         [INSTR_HDR_EXTRACT] = instr_io_export,
10772         [INSTR_HDR_EXTRACT2] = instr_io_export,
10773         [INSTR_HDR_EXTRACT3] = instr_io_export,
10774         [INSTR_HDR_EXTRACT4] = instr_io_export,
10775         [INSTR_HDR_EXTRACT5] = instr_io_export,
10776         [INSTR_HDR_EXTRACT6] = instr_io_export,
10777         [INSTR_HDR_EXTRACT7] = instr_io_export,
10778         [INSTR_HDR_EXTRACT8] = instr_io_export,
10779
10780         [INSTR_HDR_EXTRACT_M] = instr_io_export,
10781
10782         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10783
10784         [INSTR_HDR_EMIT] = instr_io_export,
10785         [INSTR_HDR_EMIT_TX] = instr_io_export,
10786         [INSTR_HDR_EMIT2_TX] = instr_io_export,
10787         [INSTR_HDR_EMIT3_TX] = instr_io_export,
10788         [INSTR_HDR_EMIT4_TX] = instr_io_export,
10789         [INSTR_HDR_EMIT5_TX] = instr_io_export,
10790         [INSTR_HDR_EMIT6_TX] = instr_io_export,
10791         [INSTR_HDR_EMIT7_TX] = instr_io_export,
10792         [INSTR_HDR_EMIT8_TX] = instr_io_export,
10793
10794         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10795         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10796
10797         [INSTR_MOV] = instr_mov_export,
10798         [INSTR_MOV_MH] = instr_mov_export,
10799         [INSTR_MOV_HM] = instr_mov_export,
10800         [INSTR_MOV_HH] = instr_mov_export,
10801         [INSTR_MOV_I] = instr_mov_export,
10802
10803         [INSTR_DMA_HT]  = instr_dma_ht_export,
10804         [INSTR_DMA_HT2] = instr_dma_ht_export,
10805         [INSTR_DMA_HT3] = instr_dma_ht_export,
10806         [INSTR_DMA_HT4] = instr_dma_ht_export,
10807         [INSTR_DMA_HT5] = instr_dma_ht_export,
10808         [INSTR_DMA_HT6] = instr_dma_ht_export,
10809         [INSTR_DMA_HT7] = instr_dma_ht_export,
10810         [INSTR_DMA_HT8] = instr_dma_ht_export,
10811
10812         [INSTR_ALU_ADD] = instr_alu_export,
10813         [INSTR_ALU_ADD_MH] = instr_alu_export,
10814         [INSTR_ALU_ADD_HM] = instr_alu_export,
10815         [INSTR_ALU_ADD_HH] = instr_alu_export,
10816         [INSTR_ALU_ADD_MI] = instr_alu_export,
10817         [INSTR_ALU_ADD_HI] = instr_alu_export,
10818
10819         [INSTR_ALU_SUB] = instr_alu_export,
10820         [INSTR_ALU_SUB_MH] = instr_alu_export,
10821         [INSTR_ALU_SUB_HM] = instr_alu_export,
10822         [INSTR_ALU_SUB_HH] = instr_alu_export,
10823         [INSTR_ALU_SUB_MI] = instr_alu_export,
10824         [INSTR_ALU_SUB_HI] = instr_alu_export,
10825
10826         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10827         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10828         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10829         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10830
10831         [INSTR_ALU_AND] = instr_alu_export,
10832         [INSTR_ALU_AND_MH] = instr_alu_export,
10833         [INSTR_ALU_AND_HM] = instr_alu_export,
10834         [INSTR_ALU_AND_HH] = instr_alu_export,
10835         [INSTR_ALU_AND_I] = instr_alu_export,
10836
10837         [INSTR_ALU_OR] = instr_alu_export,
10838         [INSTR_ALU_OR_MH] = instr_alu_export,
10839         [INSTR_ALU_OR_HM] = instr_alu_export,
10840         [INSTR_ALU_OR_HH] = instr_alu_export,
10841         [INSTR_ALU_OR_I] = instr_alu_export,
10842
10843         [INSTR_ALU_XOR] = instr_alu_export,
10844         [INSTR_ALU_XOR_MH] = instr_alu_export,
10845         [INSTR_ALU_XOR_HM] = instr_alu_export,
10846         [INSTR_ALU_XOR_HH] = instr_alu_export,
10847         [INSTR_ALU_XOR_I] = instr_alu_export,
10848
10849         [INSTR_ALU_SHL] = instr_alu_export,
10850         [INSTR_ALU_SHL_MH] = instr_alu_export,
10851         [INSTR_ALU_SHL_HM] = instr_alu_export,
10852         [INSTR_ALU_SHL_HH] = instr_alu_export,
10853         [INSTR_ALU_SHL_MI] = instr_alu_export,
10854         [INSTR_ALU_SHL_HI] = instr_alu_export,
10855
10856         [INSTR_ALU_SHR] = instr_alu_export,
10857         [INSTR_ALU_SHR_MH] = instr_alu_export,
10858         [INSTR_ALU_SHR_HM] = instr_alu_export,
10859         [INSTR_ALU_SHR_HH] = instr_alu_export,
10860         [INSTR_ALU_SHR_MI] = instr_alu_export,
10861         [INSTR_ALU_SHR_HI] = instr_alu_export,
10862
10863         [INSTR_REGPREFETCH_RH] = instr_reg_export,
10864         [INSTR_REGPREFETCH_RM] = instr_reg_export,
10865         [INSTR_REGPREFETCH_RI] = instr_reg_export,
10866
10867         [INSTR_REGRD_HRH] = instr_reg_export,
10868         [INSTR_REGRD_HRM] = instr_reg_export,
10869         [INSTR_REGRD_MRH] = instr_reg_export,
10870         [INSTR_REGRD_MRM] = instr_reg_export,
10871         [INSTR_REGRD_HRI] = instr_reg_export,
10872         [INSTR_REGRD_MRI] = instr_reg_export,
10873
10874         [INSTR_REGWR_RHH] = instr_reg_export,
10875         [INSTR_REGWR_RHM] = instr_reg_export,
10876         [INSTR_REGWR_RMH] = instr_reg_export,
10877         [INSTR_REGWR_RMM] = instr_reg_export,
10878         [INSTR_REGWR_RHI] = instr_reg_export,
10879         [INSTR_REGWR_RMI] = instr_reg_export,
10880         [INSTR_REGWR_RIH] = instr_reg_export,
10881         [INSTR_REGWR_RIM] = instr_reg_export,
10882         [INSTR_REGWR_RII] = instr_reg_export,
10883
10884         [INSTR_REGADD_RHH] = instr_reg_export,
10885         [INSTR_REGADD_RHM] = instr_reg_export,
10886         [INSTR_REGADD_RMH] = instr_reg_export,
10887         [INSTR_REGADD_RMM] = instr_reg_export,
10888         [INSTR_REGADD_RHI] = instr_reg_export,
10889         [INSTR_REGADD_RMI] = instr_reg_export,
10890         [INSTR_REGADD_RIH] = instr_reg_export,
10891         [INSTR_REGADD_RIM] = instr_reg_export,
10892         [INSTR_REGADD_RII] = instr_reg_export,
10893
10894         [INSTR_METPREFETCH_H] = instr_meter_export,
10895         [INSTR_METPREFETCH_M] = instr_meter_export,
10896         [INSTR_METPREFETCH_I] = instr_meter_export,
10897
10898         [INSTR_METER_HHM] = instr_meter_export,
10899         [INSTR_METER_HHI] = instr_meter_export,
10900         [INSTR_METER_HMM] = instr_meter_export,
10901         [INSTR_METER_HMI] = instr_meter_export,
10902         [INSTR_METER_MHM] = instr_meter_export,
10903         [INSTR_METER_MHI] = instr_meter_export,
10904         [INSTR_METER_MMM] = instr_meter_export,
10905         [INSTR_METER_MMI] = instr_meter_export,
10906         [INSTR_METER_IHM] = instr_meter_export,
10907         [INSTR_METER_IHI] = instr_meter_export,
10908         [INSTR_METER_IMM] = instr_meter_export,
10909         [INSTR_METER_IMI] = instr_meter_export,
10910
10911         [INSTR_TABLE] = instr_table_export,
10912         [INSTR_TABLE_AF] = instr_table_export,
10913         [INSTR_SELECTOR] = instr_table_export,
10914         [INSTR_LEARNER] = instr_table_export,
10915         [INSTR_LEARNER_AF] = instr_table_export,
10916
10917         [INSTR_LEARNER_LEARN] = instr_learn_export,
10918         [INSTR_LEARNER_FORGET] = instr_forget_export,
10919
10920         [INSTR_EXTERN_OBJ] = instr_extern_export,
10921         [INSTR_EXTERN_FUNC] = instr_extern_export,
10922
10923         [INSTR_JMP] = instr_jmp_export,
10924         [INSTR_JMP_VALID] = instr_jmp_export,
10925         [INSTR_JMP_INVALID] = instr_jmp_export,
10926         [INSTR_JMP_HIT] = instr_jmp_export,
10927         [INSTR_JMP_MISS] = instr_jmp_export,
10928         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
10929         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
10930
10931         [INSTR_JMP_EQ] = instr_jmp_export,
10932         [INSTR_JMP_EQ_MH] = instr_jmp_export,
10933         [INSTR_JMP_EQ_HM] = instr_jmp_export,
10934         [INSTR_JMP_EQ_HH] = instr_jmp_export,
10935         [INSTR_JMP_EQ_I] = instr_jmp_export,
10936
10937         [INSTR_JMP_NEQ] = instr_jmp_export,
10938         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
10939         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
10940         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
10941         [INSTR_JMP_NEQ_I] = instr_jmp_export,
10942
10943         [INSTR_JMP_LT] = instr_jmp_export,
10944         [INSTR_JMP_LT_MH] = instr_jmp_export,
10945         [INSTR_JMP_LT_HM] = instr_jmp_export,
10946         [INSTR_JMP_LT_HH] = instr_jmp_export,
10947         [INSTR_JMP_LT_MI] = instr_jmp_export,
10948         [INSTR_JMP_LT_HI] = instr_jmp_export,
10949
10950         [INSTR_JMP_GT] = instr_jmp_export,
10951         [INSTR_JMP_GT_MH] = instr_jmp_export,
10952         [INSTR_JMP_GT_HM] = instr_jmp_export,
10953         [INSTR_JMP_GT_HH] = instr_jmp_export,
10954         [INSTR_JMP_GT_MI] = instr_jmp_export,
10955         [INSTR_JMP_GT_HI] = instr_jmp_export,
10956
10957         [INSTR_RETURN] = instr_return_export,
10958 };
10959
10960 static void
10961 action_data_codegen(struct action *a, FILE *f)
10962 {
10963         uint32_t i;
10964
10965         fprintf(f,
10966                 "static const struct instruction action_%s_instructions[] = {\n",
10967                 a->name);
10968
10969         for (i = 0; i < a->n_instructions; i++) {
10970                 struct instruction *instr = &a->instructions[i];
10971                 instruction_export_t func = export_table[instr->type];
10972
10973                 func(instr, f);
10974         }
10975
10976         fprintf(f, "};\n");
10977 }
10978
10979 static const char *
10980 instr_type_to_func(struct instruction *instr)
10981 {
10982         switch (instr->type) {
10983         case INSTR_RX: return NULL;
10984
10985         case INSTR_TX: return "__instr_tx_exec";
10986         case INSTR_TX_I: return "__instr_tx_i_exec";
10987
10988         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
10989         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
10990         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
10991         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
10992         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
10993         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
10994         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
10995         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
10996
10997         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
10998
10999         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11000
11001         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11002         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11003         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11004         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11005         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11006         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11007         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11008         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11009         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11010
11011         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11012         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11013
11014         case INSTR_MOV: return "__instr_mov_exec";
11015         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11016         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11017         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11018         case INSTR_MOV_I: return "__instr_mov_i_exec";
11019
11020         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11021         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11022         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11023         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11024         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11025         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11026         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11027         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11028
11029         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11030         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11031         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11032         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11033         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11034         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11035
11036         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11037         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11038         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11039         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11040         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11041         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11042
11043         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11044         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11045         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11046         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11047
11048         case INSTR_ALU_AND: return "__instr_alu_and_exec";
11049         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11050         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11051         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11052         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11053
11054         case INSTR_ALU_OR: return "__instr_alu_or_exec";
11055         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11056         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11057         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11058         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11059
11060         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11061         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11062         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11063         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11064         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11065
11066         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11067         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11068         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11069         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11070         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11071         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11072
11073         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11074         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11075         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11076         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11077         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11078         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11079
11080         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11081         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11082         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11083
11084         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11085         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11086         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11087         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11088         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11089         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11090
11091         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11092         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11093         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11094         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11095         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11096         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11097         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11098         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11099         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11100
11101         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11102         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11103         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11104         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11105         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11106         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11107         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11108         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11109         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11110
11111         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11112         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11113         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11114
11115         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11116         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11117         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11118         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11119         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11120         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11121         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11122         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11123         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11124         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11125         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11126         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11127
11128         case INSTR_TABLE: return NULL;
11129         case INSTR_TABLE_AF: return NULL;
11130         case INSTR_SELECTOR: return NULL;
11131         case INSTR_LEARNER: return NULL;
11132         case INSTR_LEARNER_AF: return NULL;
11133
11134         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11135         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11136
11137         case INSTR_EXTERN_OBJ: return NULL;
11138         case INSTR_EXTERN_FUNC: return NULL;
11139
11140         case INSTR_JMP: return NULL;
11141         case INSTR_JMP_VALID: return NULL;
11142         case INSTR_JMP_INVALID: return NULL;
11143         case INSTR_JMP_HIT: return NULL;
11144         case INSTR_JMP_MISS: return NULL;
11145         case INSTR_JMP_ACTION_HIT: return NULL;
11146         case INSTR_JMP_ACTION_MISS: return NULL;
11147         case INSTR_JMP_EQ: return NULL;
11148         case INSTR_JMP_EQ_MH: return NULL;
11149         case INSTR_JMP_EQ_HM: return NULL;
11150         case INSTR_JMP_EQ_HH: return NULL;
11151         case INSTR_JMP_EQ_I: return NULL;
11152         case INSTR_JMP_NEQ: return NULL;
11153         case INSTR_JMP_NEQ_MH: return NULL;
11154         case INSTR_JMP_NEQ_HM: return NULL;
11155         case INSTR_JMP_NEQ_HH: return NULL;
11156         case INSTR_JMP_NEQ_I: return NULL;
11157         case INSTR_JMP_LT: return NULL;
11158         case INSTR_JMP_LT_MH: return NULL;
11159         case INSTR_JMP_LT_HM: return NULL;
11160         case INSTR_JMP_LT_HH: return NULL;
11161         case INSTR_JMP_LT_MI: return NULL;
11162         case INSTR_JMP_LT_HI: return NULL;
11163         case INSTR_JMP_GT: return NULL;
11164         case INSTR_JMP_GT_MH: return NULL;
11165         case INSTR_JMP_GT_HM: return NULL;
11166         case INSTR_JMP_GT_HH: return NULL;
11167         case INSTR_JMP_GT_MI: return NULL;
11168         case INSTR_JMP_GT_HI: return NULL;
11169
11170         case INSTR_RETURN: return NULL;
11171
11172         default: return NULL;
11173         }
11174 }
11175
11176 static void
11177 action_instr_does_tx_codegen(struct action *a,
11178                         uint32_t instr_pos,
11179                         struct instruction *instr,
11180                         FILE *f)
11181 {
11182         fprintf(f,
11183                 "%s(p, t, &action_%s_instructions[%u]);\n"
11184                 "\tthread_ip_reset(p, t);\n"
11185                 "\tinstr_rx_exec(p);\n"
11186                 "\treturn;\n",
11187                 instr_type_to_func(instr),
11188                 a->name,
11189                 instr_pos);
11190 }
11191
11192 static void
11193 action_instr_extern_obj_codegen(struct action *a,
11194                                 uint32_t instr_pos,
11195                                 FILE *f)
11196 {
11197         fprintf(f,
11198                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11199                 a->name,
11200                 instr_pos);
11201 }
11202
11203 static void
11204 action_instr_extern_func_codegen(struct action *a,
11205                                  uint32_t instr_pos,
11206                                  FILE *f)
11207 {
11208         fprintf(f,
11209                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11210                 a->name,
11211                 instr_pos);
11212 }
11213
11214 static void
11215 action_instr_jmp_codegen(struct action *a,
11216                          uint32_t instr_pos,
11217                          struct instruction *instr,
11218                          struct instruction_data *data,
11219                          FILE *f)
11220 {
11221         switch (instr->type) {
11222         case INSTR_JMP:
11223                 fprintf(f,
11224                         "goto %s;\n",
11225                         data->jmp_label);
11226                 return;
11227
11228         case INSTR_JMP_VALID:
11229                 fprintf(f,
11230                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11231                         "\t\tgoto %s;\n",
11232                         a->name,
11233                         instr_pos,
11234                         data->jmp_label);
11235                 return;
11236
11237         case INSTR_JMP_INVALID:
11238                 fprintf(f,
11239                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11240                         "\t\tgoto %s;\n",
11241                         a->name,
11242                         instr_pos,
11243                         data->jmp_label);
11244                 return;
11245
11246         case INSTR_JMP_HIT:
11247                 fprintf(f,
11248                         "if (t->hit)\n"
11249                         "\t\tgoto %s;\n",
11250                         data->jmp_label);
11251                 return;
11252
11253         case INSTR_JMP_MISS:
11254                 fprintf(f,
11255                         "if (!t->hit)\n"
11256                         "\t\tgoto %s;\n",
11257                         data->jmp_label);
11258                 return;
11259
11260         case INSTR_JMP_ACTION_HIT:
11261                 fprintf(f,
11262                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11263                         "\t\tgoto %s;\n",
11264                         a->name,
11265                         instr_pos,
11266                         data->jmp_label);
11267                 return;
11268
11269         case INSTR_JMP_ACTION_MISS:
11270                 fprintf(f,
11271                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11272                         "\t\tgoto %s;\n",
11273                         a->name,
11274                         instr_pos,
11275                         data->jmp_label);
11276                 return;
11277
11278         case INSTR_JMP_EQ:
11279                 fprintf(f,
11280                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11281                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11282                         "\t\tgoto %s;\n",
11283                         a->name,
11284                         instr_pos,
11285                         a->name,
11286                         instr_pos,
11287                         data->jmp_label);
11288                 return;
11289
11290         case INSTR_JMP_EQ_MH:
11291                 fprintf(f,
11292                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11293                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11294                         "\t\tgoto %s;\n",
11295                         a->name,
11296                         instr_pos,
11297                         a->name,
11298                         instr_pos,
11299                         data->jmp_label);
11300                 return;
11301
11302         case INSTR_JMP_EQ_HM:
11303                 fprintf(f,
11304                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11305                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11306                         "\t\tgoto %s;\n",
11307                         a->name,
11308                         instr_pos,
11309                         a->name,
11310                         instr_pos,
11311                         data->jmp_label);
11312                 return;
11313
11314         case INSTR_JMP_EQ_HH:
11315                 fprintf(f,
11316                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11317                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11318                         "\t\tgoto %s;\n",
11319                         a->name,
11320                         instr_pos,
11321                         a->name,
11322                         instr_pos,
11323                         data->jmp_label);
11324                 return;
11325
11326         case INSTR_JMP_EQ_I:
11327                 fprintf(f,
11328                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11329                         "action_%s_instructions[%u].jmp.b_val)\n"
11330                         "\t\tgoto %s;\n",
11331                         a->name,
11332                         instr_pos,
11333                         a->name,
11334                         instr_pos,
11335                         data->jmp_label);
11336                 return;
11337
11338         case INSTR_JMP_NEQ:
11339                 fprintf(f,
11340                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11341                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11342                         "\t\tgoto %s;\n",
11343                         a->name,
11344                         instr_pos,
11345                         a->name,
11346                         instr_pos,
11347                         data->jmp_label);
11348                 return;
11349
11350         case INSTR_JMP_NEQ_MH:
11351                 fprintf(f,
11352                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11353                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11354                         "\t\tgoto %s;\n",
11355                         a->name,
11356                         instr_pos,
11357                         a->name,
11358                         instr_pos,
11359                         data->jmp_label);
11360                 return;
11361
11362         case INSTR_JMP_NEQ_HM:
11363                 fprintf(f,
11364                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11365                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11366                         "\t\tgoto %s;\n",
11367                         a->name,
11368                         instr_pos,
11369                         a->name,
11370                         instr_pos,
11371                         data->jmp_label);
11372                 return;
11373
11374         case INSTR_JMP_NEQ_HH:
11375                 fprintf(f,
11376                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11377                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11378                         "\t\tgoto %s;\n",
11379                         a->name,
11380                         instr_pos,
11381                         a->name,
11382                         instr_pos,
11383                         data->jmp_label);
11384                 return;
11385
11386         case INSTR_JMP_NEQ_I:
11387                 fprintf(f,
11388                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11389                         "action_%s_instructions[%u].jmp.b_val)\n"
11390                         "\t\tgoto %s;\n",
11391                         a->name,
11392                         instr_pos,
11393                         a->name,
11394                         instr_pos,
11395                         data->jmp_label);
11396                 return;
11397
11398         case INSTR_JMP_LT:
11399                 fprintf(f,
11400                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11401                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11402                         "\t\tgoto %s;\n",
11403                         a->name,
11404                         instr_pos,
11405                         a->name,
11406                         instr_pos,
11407                         data->jmp_label);
11408                 return;
11409
11410         case INSTR_JMP_LT_MH:
11411                 fprintf(f,
11412                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11413                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11414                         "\t\tgoto %s;\n",
11415                         a->name,
11416                         instr_pos,
11417                         a->name,
11418                         instr_pos,
11419                         data->jmp_label);
11420                 return;
11421
11422         case INSTR_JMP_LT_HM:
11423                 fprintf(f,
11424                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11425                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11426                         "\t\tgoto %s;\n",
11427                         a->name,
11428                         instr_pos,
11429                         a->name,
11430                         instr_pos,
11431                         data->jmp_label);
11432                 return;
11433
11434         case INSTR_JMP_LT_HH:
11435                 fprintf(f,
11436                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11437                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11438                         "\t\tgoto %s;\n",
11439                         a->name,
11440                         instr_pos,
11441                         a->name,
11442                         instr_pos,
11443                         data->jmp_label);
11444                 return;
11445
11446         case INSTR_JMP_LT_MI:
11447                 fprintf(f,
11448                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11449                         "action_%s_instructions[%u].jmp.b_val)\n"
11450                         "\t\tgoto %s;\n",
11451                         a->name,
11452                         instr_pos,
11453                         a->name,
11454                         instr_pos,
11455                         data->jmp_label);
11456                 return;
11457
11458         case INSTR_JMP_LT_HI:
11459                 fprintf(f,
11460                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11461                         "action_%s_instructions[%u].jmp.b_val)\n"
11462                         "\t\tgoto %s;\n",
11463                         a->name,
11464                         instr_pos,
11465                         a->name,
11466                         instr_pos,
11467                         data->jmp_label);
11468                 return;
11469
11470         case INSTR_JMP_GT:
11471                 fprintf(f,
11472                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11473                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11474                         "\t\tgoto %s;\n",
11475                         a->name,
11476                         instr_pos,
11477                         a->name,
11478                         instr_pos,
11479                         data->jmp_label);
11480                 return;
11481
11482         case INSTR_JMP_GT_MH:
11483                 fprintf(f,
11484                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11485                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11486                         "\t\tgoto %s;\n",
11487                         a->name,
11488                         instr_pos,
11489                         a->name,
11490                         instr_pos,
11491                         data->jmp_label);
11492                 return;
11493
11494         case INSTR_JMP_GT_HM:
11495                 fprintf(f,
11496                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11497                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11498                         "\t\tgoto %s;\n",
11499                         a->name,
11500                         instr_pos,
11501                         a->name,
11502                         instr_pos,
11503                         data->jmp_label);
11504                 return;
11505
11506         case INSTR_JMP_GT_HH:
11507                 fprintf(f,
11508                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11509                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11510                         "\t\tgoto %s;\n",
11511                         a->name,
11512                         instr_pos,
11513                         a->name,
11514                         instr_pos,
11515                         data->jmp_label);
11516                 return;
11517
11518         case INSTR_JMP_GT_MI:
11519                 fprintf(f,
11520                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11521                         "action_%s_instructions[%u].jmp.b_val)\n"
11522                         "\t\tgoto %s;\n",
11523                         a->name,
11524                         instr_pos,
11525                         a->name,
11526                         instr_pos,
11527                         data->jmp_label);
11528                 return;
11529
11530         case INSTR_JMP_GT_HI:
11531                 fprintf(f,
11532                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11533                         "action_%s_instructions[%u].jmp.b_val)\n"
11534                         "\t\tgoto %s;\n",
11535                         a->name,
11536                         instr_pos,
11537                         a->name,
11538                         instr_pos,
11539                         data->jmp_label);
11540                 return;
11541
11542         default:
11543                 return;
11544         }
11545 }
11546
11547 static void
11548 action_instr_return_codegen(FILE *f)
11549 {
11550         fprintf(f,
11551                 "return;\n");
11552 }
11553
11554 static void
11555 action_instr_codegen(struct action *a, FILE *f)
11556 {
11557         uint32_t i;
11558
11559         fprintf(f,
11560                 "void\n"
11561                 "action_%s_run(struct rte_swx_pipeline *p)\n"
11562                 "{\n"
11563                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11564                 "\n",
11565                 a->name);
11566
11567         for (i = 0; i < a->n_instructions; i++) {
11568                 struct instruction *instr = &a->instructions[i];
11569                 struct instruction_data *data = &a->instruction_data[i];
11570
11571                 /* Label, if present. */
11572                 if (data->label[0])
11573                         fprintf(f, "\n%s : ", data->label);
11574                 else
11575                         fprintf(f, "\n\t");
11576
11577                 /* TX instruction type. */
11578                 if (instruction_does_tx(instr)) {
11579                         action_instr_does_tx_codegen(a, i, instr, f);
11580                         continue;
11581                 }
11582
11583                 /* Extern object/function instruction type. */
11584                 if (instr->type == INSTR_EXTERN_OBJ) {
11585                         action_instr_extern_obj_codegen(a, i, f);
11586                         continue;
11587                 }
11588
11589                 if (instr->type == INSTR_EXTERN_FUNC) {
11590                         action_instr_extern_func_codegen(a, i, f);
11591                         continue;
11592                 }
11593
11594                 /* Jump instruction type. */
11595                 if (instruction_is_jmp(instr)) {
11596                         action_instr_jmp_codegen(a, i, instr, data, f);
11597                         continue;
11598                 }
11599
11600                 /* Return instruction type. */
11601                 if (instr->type == INSTR_RETURN) {
11602                         action_instr_return_codegen(f);
11603                         continue;
11604                 }
11605
11606                 /* Any other instruction type. */
11607                 fprintf(f,
11608                         "%s(p, t, &action_%s_instructions[%u]);\n",
11609                         instr_type_to_func(instr),
11610                         a->name,
11611                         i);
11612         }
11613
11614         fprintf(f, "}\n\n");
11615 }
11616
11617 struct instruction_group {
11618         TAILQ_ENTRY(instruction_group) node;
11619
11620         uint32_t group_id;
11621
11622         uint32_t first_instr_id;
11623
11624         uint32_t last_instr_id;
11625
11626         instr_exec_t func;
11627 };
11628
11629 TAILQ_HEAD(instruction_group_list, instruction_group);
11630
11631 static struct instruction_group *
11632 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11633 {
11634         struct instruction_group *g;
11635
11636         TAILQ_FOREACH(g, igl, node)
11637                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11638                         return g;
11639
11640         return NULL;
11641 }
11642
11643 static void
11644 instruction_group_list_free(struct instruction_group_list *igl)
11645 {
11646         if (!igl)
11647                 return;
11648
11649         for ( ; ; ) {
11650                 struct instruction_group *g;
11651
11652                 g = TAILQ_FIRST(igl);
11653                 if (!g)
11654                         break;
11655
11656                 TAILQ_REMOVE(igl, g, node);
11657                 free(g);
11658         }
11659
11660         free(igl);
11661 }
11662
11663 static struct instruction_group_list *
11664 instruction_group_list_create(struct rte_swx_pipeline *p)
11665 {
11666         struct instruction_group_list *igl = NULL;
11667         struct instruction_group *g = NULL;
11668         uint32_t n_groups = 0, i;
11669
11670         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11671                 goto error;
11672
11673         /* List init. */
11674         igl = calloc(1, sizeof(struct instruction_group_list));
11675         if (!igl)
11676                 goto error;
11677
11678         TAILQ_INIT(igl);
11679
11680         /* Allocate the first group. */
11681         g = calloc(1, sizeof(struct instruction_group));
11682         if (!g)
11683                 goto error;
11684
11685         /* Iteration 1: Separate the instructions into groups based on the thread yield
11686          * instructions. Do not worry about the jump instructions at this point.
11687          */
11688         for (i = 0; i < p->n_instructions; i++) {
11689                 struct instruction *instr = &p->instructions[i];
11690
11691                 /* Check for thread yield instructions. */
11692                 if (!instruction_does_thread_yield(instr))
11693                         continue;
11694
11695                 /* If the current group contains at least one instruction, then finalize it (with
11696                  * the previous instruction), add it to the list and allocate a new group (that
11697                  * starts with the current instruction).
11698                  */
11699                 if (i - g->first_instr_id) {
11700                         /* Finalize the group. */
11701                         g->last_instr_id = i - 1;
11702
11703                         /* Add the group to the list. Advance the number of groups. */
11704                         TAILQ_INSERT_TAIL(igl, g, node);
11705                         n_groups++;
11706
11707                         /* Allocate a new group. */
11708                         g = calloc(1, sizeof(struct instruction_group));
11709                         if (!g)
11710                                 goto error;
11711
11712                         /* Initialize the new group. */
11713                         g->group_id = n_groups;
11714                         g->first_instr_id = i;
11715                 }
11716
11717                 /* Finalize the current group (with the current instruction, therefore this group
11718                  * contains just the current thread yield instruction), add it to the list and
11719                  * allocate a new group (that starts with the next instruction).
11720                  */
11721
11722                 /* Finalize the group. */
11723                 g->last_instr_id = i;
11724
11725                 /* Add the group to the list. Advance the number of groups. */
11726                 TAILQ_INSERT_TAIL(igl, g, node);
11727                 n_groups++;
11728
11729                 /* Allocate a new group. */
11730                 g = calloc(1, sizeof(struct instruction_group));
11731                 if (!g)
11732                         goto error;
11733
11734                 /* Initialize the new group. */
11735                 g->group_id = n_groups;
11736                 g->first_instr_id = i + 1;
11737         }
11738
11739         /* Handle the last group. */
11740         if (i - g->first_instr_id) {
11741                 /* Finalize the group. */
11742                 g->last_instr_id = i - 1;
11743
11744                 /* Add the group to the list. Advance the number of groups. */
11745                 TAILQ_INSERT_TAIL(igl, g, node);
11746                 n_groups++;
11747         } else
11748                 free(g);
11749
11750         g = NULL;
11751
11752         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11753          * the destination of a jump instruction located in a different group ("far jump"), then the
11754          * current group has to be split, so that the instruction representing the far jump
11755          * destination is at the start of its group.
11756          */
11757         for ( ; ; ) {
11758                 int is_modified = 0;
11759
11760                 for (i = 0; i < p->n_instructions; i++) {
11761                         struct instruction_data *data = &p->instruction_data[i];
11762                         struct instruction_group *g;
11763                         uint32_t j;
11764
11765                         /* Continue when the current instruction is not a jump destination. */
11766                         if (!data->n_users)
11767                                 continue;
11768
11769                         g = instruction_group_list_group_find(igl, i);
11770                         if (!g)
11771                                 goto error;
11772
11773                         /* Find out all the jump instructions with this destination. */
11774                         for (j = 0; j < p->n_instructions; j++) {
11775                                 struct instruction *jmp_instr = &p->instructions[j];
11776                                 struct instruction_data *jmp_data = &p->instruction_data[j];
11777                                 struct instruction_group *jmp_g, *new_g;
11778
11779                                 /* Continue when not a jump instruction. Even when jump instruction,
11780                                  * continue when the jump destination is not this instruction.
11781                                  */
11782                                 if (!instruction_is_jmp(jmp_instr) ||
11783                                     strcmp(jmp_data->jmp_label, data->label))
11784                                         continue;
11785
11786                                 jmp_g = instruction_group_list_group_find(igl, j);
11787                                 if (!jmp_g)
11788                                         goto error;
11789
11790                                 /* Continue when both the jump instruction and the jump destination
11791                                  * instruction are in the same group. Even when in different groups,
11792                                  * still continue if the jump destination instruction is already the
11793                                  * first instruction of its group.
11794                                  */
11795                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11796                                         continue;
11797
11798                                 /* Split the group of the current jump destination instruction to
11799                                  * make this instruction the first instruction of a new group.
11800                                  */
11801                                 new_g = calloc(1, sizeof(struct instruction_group));
11802                                 if (!new_g)
11803                                         goto error;
11804
11805                                 new_g->group_id = n_groups;
11806                                 new_g->first_instr_id = i;
11807                                 new_g->last_instr_id = g->last_instr_id;
11808
11809                                 g->last_instr_id = i - 1;
11810
11811                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11812                                 n_groups++;
11813                                 is_modified = 1;
11814
11815                                 /* The decision to split this group (to make the current instruction
11816                                  * the first instruction of a new group) is already taken and fully
11817                                  * implemented, so no need to search for more reasons to do it.
11818                                  */
11819                                 break;
11820                         }
11821                 }
11822
11823                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11824                  * previously considered local (i.e. the jump destination is in the same group as
11825                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11826                  * different group than the jump instruction). Wost case scenario: each instruction
11827                  * that is a jump destination ends up as the first instruction of its group.
11828                  */
11829                 if (!is_modified)
11830                         break;
11831         }
11832
11833         /* Re-assign the group IDs to be in incremental order. */
11834         i = 0;
11835         TAILQ_FOREACH(g, igl, node) {
11836                 g->group_id = i;
11837
11838                 i++;
11839         }
11840
11841         return igl;
11842
11843 error:
11844         instruction_group_list_free(igl);
11845
11846         free(g);
11847
11848         return NULL;
11849 }
11850
11851 static void
11852 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11853                                uint32_t instr_pos,
11854                                struct instruction *instr,
11855                                FILE *f)
11856 {
11857         fprintf(f,
11858                 "%s(p, t, &pipeline_instructions[%u]);\n"
11859                 "\tthread_ip_reset(p, t);\n"
11860                 "\tinstr_rx_exec(p);\n"
11861                 "\treturn;\n",
11862                 instr_type_to_func(instr),
11863                 instr_pos);
11864 }
11865
11866 static int
11867 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11868                            struct instruction_group_list *igl,
11869                            uint32_t jmp_instr_id,
11870                            struct instruction *jmp_instr,
11871                            struct instruction_data *jmp_data,
11872                            FILE *f)
11873 {
11874         struct instruction_group *jmp_g, *g;
11875         struct instruction_data *data;
11876         uint32_t instr_id;
11877
11878         switch (jmp_instr->type) {
11879         case INSTR_JMP:
11880                 break;
11881
11882         case INSTR_JMP_VALID:
11883                 fprintf(f,
11884                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11885                         jmp_instr_id);
11886                 break;
11887
11888         case INSTR_JMP_INVALID:
11889                 fprintf(f,
11890                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11891                         jmp_instr_id);
11892                 break;
11893
11894         case INSTR_JMP_HIT:
11895                 fprintf(f,
11896                         "if (t->hit)\n");
11897                 break;
11898
11899         case INSTR_JMP_MISS:
11900                 fprintf(f,
11901                         "if (!t->hit)\n");
11902                 break;
11903
11904         case INSTR_JMP_ACTION_HIT:
11905                 fprintf(f,
11906                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
11907                         jmp_instr_id);
11908                 break;
11909
11910         case INSTR_JMP_ACTION_MISS:
11911                 fprintf(f,
11912                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
11913                         jmp_instr_id);
11914                 break;
11915
11916         case INSTR_JMP_EQ:
11917                 fprintf(f,
11918                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11919                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11920                         jmp_instr_id,
11921                         jmp_instr_id);
11922                 break;
11923
11924         case INSTR_JMP_EQ_MH:
11925                 fprintf(f,
11926                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11927                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11928                         jmp_instr_id,
11929                         jmp_instr_id);
11930                 break;
11931
11932         case INSTR_JMP_EQ_HM:
11933                 fprintf(f,
11934                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11935                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11936                         jmp_instr_id,
11937                         jmp_instr_id);
11938                 break;
11939
11940         case INSTR_JMP_EQ_HH:
11941                 fprintf(f,
11942                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11943                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11944                         jmp_instr_id,
11945                         jmp_instr_id);
11946                 break;
11947
11948         case INSTR_JMP_EQ_I:
11949                 fprintf(f,
11950                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11951                         "pipeline_instructions[%u].jmp.b_val)",
11952                         jmp_instr_id,
11953                         jmp_instr_id);
11954                 break;
11955
11956         case INSTR_JMP_NEQ:
11957                 fprintf(f,
11958                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11959                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11960                         jmp_instr_id,
11961                         jmp_instr_id);
11962                 break;
11963
11964         case INSTR_JMP_NEQ_MH:
11965                 fprintf(f,
11966                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11967                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11968                         jmp_instr_id,
11969                         jmp_instr_id);
11970                 break;
11971
11972         case INSTR_JMP_NEQ_HM:
11973                 fprintf(f,
11974                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11975                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11976                         jmp_instr_id,
11977                         jmp_instr_id);
11978                 break;
11979
11980         case INSTR_JMP_NEQ_HH:
11981                 fprintf(f,
11982                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11983                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11984                         jmp_instr_id,
11985                         jmp_instr_id);
11986                 break;
11987
11988         case INSTR_JMP_NEQ_I:
11989                 fprintf(f,
11990                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11991                         "pipeline_instructions[%u].jmp.b_val)",
11992                         jmp_instr_id,
11993                         jmp_instr_id);
11994                 break;
11995
11996         case INSTR_JMP_LT:
11997                 fprintf(f,
11998                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11999                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12000                         jmp_instr_id,
12001                         jmp_instr_id);
12002                 break;
12003
12004         case INSTR_JMP_LT_MH:
12005                 fprintf(f,
12006                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12007                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12008                         jmp_instr_id,
12009                         jmp_instr_id);
12010                 break;
12011
12012         case INSTR_JMP_LT_HM:
12013                 fprintf(f,
12014                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12015                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12016                         jmp_instr_id,
12017                         jmp_instr_id);
12018                 break;
12019
12020         case INSTR_JMP_LT_HH:
12021                 fprintf(f,
12022                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12023                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12024                         jmp_instr_id,
12025                         jmp_instr_id);
12026                 break;
12027
12028         case INSTR_JMP_LT_MI:
12029                 fprintf(f,
12030                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12031                         "pipeline_instructions[%u].jmp.b_val)",
12032                         jmp_instr_id,
12033                         jmp_instr_id);
12034                 break;
12035
12036         case INSTR_JMP_LT_HI:
12037                 fprintf(f,
12038                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12039                         "pipeline_instructions[%u].jmp.b_val)",
12040                         jmp_instr_id,
12041                         jmp_instr_id);
12042                 break;
12043
12044         case INSTR_JMP_GT:
12045                 fprintf(f,
12046                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12047                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12048                         jmp_instr_id,
12049                         jmp_instr_id);
12050                 break;
12051
12052         case INSTR_JMP_GT_MH:
12053                 fprintf(f,
12054                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12055                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12056                         jmp_instr_id,
12057                         jmp_instr_id);
12058                 break;
12059
12060         case INSTR_JMP_GT_HM:
12061                 fprintf(f,
12062                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12063                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12064                         jmp_instr_id,
12065                         jmp_instr_id);
12066                 break;
12067
12068         case INSTR_JMP_GT_HH:
12069                 fprintf(f,
12070                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12071                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12072                         jmp_instr_id,
12073                         jmp_instr_id);
12074                 break;
12075
12076         case INSTR_JMP_GT_MI:
12077                 fprintf(f,
12078                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12079                         "pipeline_instructions[%u].jmp.b_val)",
12080                         jmp_instr_id,
12081                         jmp_instr_id);
12082                 break;
12083
12084         case INSTR_JMP_GT_HI:
12085                 fprintf(f,
12086                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12087                         "pipeline_instructions[%u].jmp.b_val)",
12088                         jmp_instr_id,
12089                         jmp_instr_id);
12090                 break;
12091
12092         default:
12093                 break;
12094         }
12095
12096         /* Find the instruction group of the jump instruction. */
12097         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12098         if (!jmp_g)
12099                 return -EINVAL;
12100
12101         /* Find the instruction group of the jump destination instruction. */
12102         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12103         if (!data)
12104                 return -EINVAL;
12105
12106         instr_id = data - p->instruction_data;
12107
12108         g = instruction_group_list_group_find(igl, instr_id);
12109         if (!g)
12110                 return -EINVAL;
12111
12112         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12113          * instruction group).
12114          */
12115         if (g->group_id == jmp_g->group_id)
12116                 fprintf(f,
12117                         "\n\t\tgoto %s;\n",
12118                         jmp_data->jmp_label);
12119         else
12120                 fprintf(f,
12121                         " {\n"
12122                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12123                         "\t\treturn;\n"
12124                         "\t}\n\n",
12125                         g->group_id);
12126
12127         return 0;
12128 }
12129
12130 static void
12131 instruction_group_list_codegen(struct instruction_group_list *igl,
12132                                struct rte_swx_pipeline *p,
12133                                FILE *f)
12134 {
12135         struct instruction_group *g;
12136         uint32_t i;
12137         int is_required = 0;
12138
12139         /* Check if code generation is required. */
12140         TAILQ_FOREACH(g, igl, node)
12141                 if (g->first_instr_id < g->last_instr_id)
12142                         is_required = 1;
12143
12144         if (!is_required)
12145                 return;
12146
12147         /* Generate the code for the pipeline instruction array. */
12148         fprintf(f,
12149                 "static const struct instruction pipeline_instructions[] = {\n");
12150
12151         for (i = 0; i < p->n_instructions; i++) {
12152                 struct instruction *instr = &p->instructions[i];
12153                 instruction_export_t func = export_table[instr->type];
12154
12155                 func(instr, f);
12156         }
12157
12158         fprintf(f, "};\n\n");
12159
12160         /* Generate the code for the pipeline functions: one function for each instruction group
12161          * that contains more than one instruction.
12162          */
12163         TAILQ_FOREACH(g, igl, node) {
12164                 struct instruction *last_instr;
12165                 uint32_t j;
12166
12167                 /* Skip if group contains a single instruction. */
12168                 if (g->last_instr_id == g->first_instr_id)
12169                         continue;
12170
12171                 /* Generate new pipeline function. */
12172                 fprintf(f,
12173                         "void\n"
12174                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12175                         "{\n"
12176                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12177                         "\n",
12178                         g->group_id);
12179
12180                 /* Generate the code for each pipeline instruction. */
12181                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12182                         struct instruction *instr = &p->instructions[j];
12183                         struct instruction_data *data = &p->instruction_data[j];
12184
12185                         /* Label, if present. */
12186                         if (data->label[0])
12187                                 fprintf(f, "\n%s : ", data->label);
12188                         else
12189                                 fprintf(f, "\n\t");
12190
12191                         /* TX instruction type. */
12192                         if (instruction_does_tx(instr)) {
12193                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12194                                 continue;
12195                         }
12196
12197                         /* Jump instruction type. */
12198                         if (instruction_is_jmp(instr)) {
12199                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12200                                 continue;
12201                         }
12202
12203                         /* Any other instruction type. */
12204                         fprintf(f,
12205                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12206                                 instr_type_to_func(instr),
12207                                 j);
12208                 }
12209
12210                 /* Finalize the generated pipeline function. For some instructions such as TX,
12211                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12212                  * decided unconditionally and the instruction pointer of the current thread set
12213                  * accordingly; for all the other instructions, the instruction pointer must be
12214                  * incremented now.
12215                  */
12216                 last_instr = &p->instructions[g->last_instr_id];
12217
12218                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12219                         fprintf(f,
12220                                 "thread_ip_inc(p);\n");
12221
12222                 fprintf(f,
12223                         "}\n"
12224                         "\n");
12225         }
12226 }
12227
12228 static uint32_t
12229 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12230 {
12231         struct instruction_group *g;
12232         uint32_t n_custom_instr = 0;
12233
12234         /* Groups with a single instruction: no function is generated for this group, the group
12235          * keeps its current instruction. Groups with more than two instructions: one function and
12236          * the associated custom instruction get generated for each such group.
12237          */
12238         TAILQ_FOREACH(g, igl, node) {
12239                 if (g->first_instr_id == g->last_instr_id)
12240                         continue;
12241
12242                 n_custom_instr++;
12243         }
12244
12245         return n_custom_instr;
12246 }
12247
12248 static int
12249 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12250 {
12251         struct action *a;
12252         FILE *f = NULL;
12253
12254         /* Create the .c file. */
12255         f = fopen("/tmp/pipeline.c", "w");
12256         if (!f)
12257                 return -EIO;
12258
12259         /* Include the .h file. */
12260         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12261
12262         /* Add the code for each action. */
12263         TAILQ_FOREACH(a, &p->actions, node) {
12264                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12265
12266                 action_data_codegen(a, f);
12267
12268                 fprintf(f, "\n");
12269
12270                 action_instr_codegen(a, f);
12271
12272                 fprintf(f, "\n");
12273         }
12274
12275         /* Add the pipeline code. */
12276         instruction_group_list_codegen(igl, p, f);
12277
12278         /* Close the .c file. */
12279         fclose(f);
12280
12281         return 0;
12282 }
12283
12284 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12285 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12286 #endif
12287
12288 static int
12289 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12290 {
12291         struct action *a;
12292         struct instruction_group *g;
12293         char *dir_in, *buffer = NULL;
12294         const char *dir_out;
12295         int status = 0;
12296
12297         /* Get the environment variables. */
12298         dir_in = getenv("RTE_INSTALL_DIR");
12299         if (!dir_in) {
12300                 status = -EINVAL;
12301                 goto free;
12302         }
12303
12304         dir_out = "/tmp";
12305
12306         /* Memory allocation for the command buffer. */
12307         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12308         if (!buffer) {
12309                 status = -ENOMEM;
12310                 goto free;
12311         }
12312
12313         snprintf(buffer,
12314                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12315                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12316                  "-I %s/lib/pipeline "
12317                  "-I %s/lib/eal/include "
12318                  "-I %s/lib/eal/x86/include "
12319                  "-I %s/lib/eal/include/generic "
12320                  "-I %s/lib/meter "
12321                  "-I %s/lib/port "
12322                  "-I %s/lib/table "
12323                  "-I %s/lib/pipeline "
12324                  "-I %s/config "
12325                  "-I %s/build "
12326                  "-I %s/lib/eal/linux/include "
12327                  ">%s/pipeline.log 2>&1 "
12328                  "&& "
12329                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12330                  ">>%s/pipeline.log 2>&1",
12331                  dir_out,
12332                  dir_out,
12333                  dir_in,
12334                  dir_in,
12335                  dir_in,
12336                  dir_in,
12337                  dir_in,
12338                  dir_in,
12339                  dir_in,
12340                  dir_in,
12341                  dir_in,
12342                  dir_in,
12343                  dir_in,
12344                  dir_out,
12345                  dir_out,
12346                  dir_out,
12347                  dir_out);
12348
12349         /* Build the shared object library. */
12350         status = system(buffer);
12351         if (status)
12352                 goto free;
12353
12354         /* Open library. */
12355         snprintf(buffer,
12356                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12357                  "%s/libpipeline.so",
12358                  dir_out);
12359
12360         p->lib = dlopen(buffer, RTLD_LAZY);
12361         if (!p->lib) {
12362                 status = -EIO;
12363                 goto free;
12364         }
12365
12366         /* Get the action function symbols. */
12367         TAILQ_FOREACH(a, &p->actions, node) {
12368                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12369
12370                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12371                 if (!p->action_funcs[a->id]) {
12372                         status = -EINVAL;
12373                         goto free;
12374                 }
12375         }
12376
12377         /* Get the pipeline function symbols. */
12378         TAILQ_FOREACH(g, igl, node) {
12379                 if (g->first_instr_id == g->last_instr_id)
12380                         continue;
12381
12382                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12383
12384                 g->func = dlsym(p->lib, buffer);
12385                 if (!g->func) {
12386                         status = -EINVAL;
12387                         goto free;
12388                 }
12389         }
12390
12391 free:
12392         if (status && p->lib) {
12393                 dlclose(p->lib);
12394                 p->lib = NULL;
12395         }
12396
12397         free(buffer);
12398
12399         return status;
12400 }
12401
12402 static int
12403 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12404                       struct instruction_group_list *igl)
12405 {
12406         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12407
12408         /* Check that enough space is available within the pipeline instruction table to store all
12409          * the custom instructions.
12410          */
12411         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12412                 return -ENOSPC;
12413
12414         return 0;
12415 }
12416
12417 static void
12418 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12419 {
12420         struct instruction_group *g;
12421         uint32_t i;
12422
12423         /* Pipeline table instructions. */
12424         for (i = 0; i < p->n_instructions; i++) {
12425                 struct instruction *instr = &p->instructions[i];
12426
12427                 if (instr->type == INSTR_TABLE)
12428                         instr->type = INSTR_TABLE_AF;
12429
12430                 if (instr->type == INSTR_LEARNER)
12431                         instr->type = INSTR_LEARNER_AF;
12432         }
12433
12434         /* Pipeline custom instructions. */
12435         i = 0;
12436         TAILQ_FOREACH(g, igl, node) {
12437                 struct instruction *instr = &p->instructions[g->first_instr_id];
12438                 uint32_t j;
12439
12440                 if (g->first_instr_id == g->last_instr_id)
12441                         continue;
12442
12443                 /* Install a new custom instruction. */
12444                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12445
12446                 /* First instruction of the group: change its type to the new custom instruction. */
12447                 instr->type = INSTR_CUSTOM_0 + i;
12448
12449                 /* All the subsequent instructions of the group: invalidate. */
12450                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12451                         struct instruction_data *data = &p->instruction_data[j];
12452
12453                         data->invalid = 1;
12454                 }
12455
12456                 i++;
12457         }
12458
12459         /* Remove the invalidated instructions. */
12460         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12461
12462         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12463          * instructions that are the only instruction within their group, so they were left
12464          * unmodified).
12465          */
12466         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12467 }
12468
12469 static int
12470 pipeline_compile(struct rte_swx_pipeline *p)
12471 {
12472         struct instruction_group_list *igl = NULL;
12473         int status = 0;
12474
12475         igl = instruction_group_list_create(p);
12476         if (!igl) {
12477                 status = -ENOMEM;
12478                 goto free;
12479         }
12480
12481         /* Code generation. */
12482         status = pipeline_codegen(p, igl);
12483         if (status)
12484                 goto free;
12485
12486         /* Build and load the shared object library. */
12487         status = pipeline_libload(p, igl);
12488         if (status)
12489                 goto free;
12490
12491         /* Adjust instructions. */
12492         status = pipeline_adjust_check(p, igl);
12493         if (status)
12494                 goto free;
12495
12496         pipeline_adjust(p, igl);
12497
12498 free:
12499         instruction_group_list_free(igl);
12500
12501         return status;
12502 }