lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8 #include <dlfcn.h>
   9
  10 #include "rte_swx_pipeline_internal.h"
  11
  12 #define CHECK(condition, err_code)                                             \
  13 do {                                                                           \
  14         if (!(condition))                                                      \
  15                 return -(err_code);                                            \
  16 } while (0)
  17
  18 #define CHECK_NAME(name, err_code)                                             \
  19         CHECK((name) &&                                                        \
  20               (name)[0] &&                                                     \
  21               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  22               err_code)
  23
  24 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  25         CHECK((instr) &&                                                       \
  26               (instr)[0] &&                                                    \
  27               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  28                RTE_SWX_INSTRUCTION_SIZE),                                      \
  29               err_code)
  30
  31 /*
  32  * Environment.
  33  */
  34 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  35
  36 #include <rte_malloc.h>
  37
  38 static void *
  39 env_malloc(size_t size, size_t alignment, int numa_node)
  40 {
  41         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  42 }
  43
  44 static void
  45 env_free(void *start, size_t size __rte_unused)
  46 {
  47         rte_free(start);
  48 }
  49
  50 #else
  51
  52 #include <numa.h>
  53
  54 static void *
  55 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  56 {
  57         void *start;
  58
  59         if (numa_available() == -1)
  60                 return NULL;
  61
  62         start = numa_alloc_onnode(size, numa_node);
  63         if (!start)
  64                 return NULL;
  65
  66         memset(start, 0, size);
  67         return start;
  68 }
  69
  70 static void
  71 env_free(void *start, size_t size)
  72 {
  73         if (numa_available() == -1)
  74                 return;
  75
  76         numa_free(start, size);
  77 }
  78
  79 #endif
  80
  81 /*
  82  * Struct.
  83  */
  84 static struct struct_type *
  85 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  86 {
  87         struct struct_type *elem;
  88
  89         TAILQ_FOREACH(elem, &p->struct_types, node)
  90                 if (strcmp(elem->name, name) == 0)
  91                         return elem;
  92
  93         return NULL;
  94 }
  95
  96 static struct field *
  97 struct_type_field_find(struct struct_type *st, const char *name)
  98 {
  99         uint32_t i;
 100
 101         for (i = 0; i < st->n_fields; i++) {
 102                 struct field *f = &st->fields[i];
 103
 104                 if (strcmp(f->name, name) == 0)
 105                         return f;
 106         }
 107
 108         return NULL;
 109 }
 110
 111 int
 112 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 113                                       const char *name,
 114                                       struct rte_swx_field_params *fields,
 115                                       uint32_t n_fields,
 116                                       int last_field_has_variable_size)
 117 {
 118         struct struct_type *st;
 119         uint32_t i;
 120
 121         CHECK(p, EINVAL);
 122         CHECK_NAME(name, EINVAL);
 123         CHECK(fields, EINVAL);
 124         CHECK(n_fields, EINVAL);
 125
 126         for (i = 0; i < n_fields; i++) {
 127                 struct rte_swx_field_params *f = &fields[i];
 128                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 129                 uint32_t j;
 130
 131                 CHECK_NAME(f->name, EINVAL);
 132                 CHECK(f->n_bits, EINVAL);
 133                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 134                 CHECK((f->n_bits & 7) == 0, EINVAL);
 135
 136                 for (j = 0; j < i; j++) {
 137                         struct rte_swx_field_params *f_prev = &fields[j];
 138
 139                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 140                 }
 141         }
 142
 143         CHECK(!struct_type_find(p, name), EEXIST);
 144
 145         /* Node allocation. */
 146         st = calloc(1, sizeof(struct struct_type));
 147         CHECK(st, ENOMEM);
 148
 149         st->fields = calloc(n_fields, sizeof(struct field));
 150         if (!st->fields) {
 151                 free(st);
 152                 CHECK(0, ENOMEM);
 153         }
 154
 155         /* Node initialization. */
 156         strcpy(st->name, name);
 157         for (i = 0; i < n_fields; i++) {
 158                 struct field *dst = &st->fields[i];
 159                 struct rte_swx_field_params *src = &fields[i];
 160                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 161
 162                 strcpy(dst->name, src->name);
 163                 dst->n_bits = src->n_bits;
 164                 dst->offset = st->n_bits;
 165                 dst->var_size = var_size;
 166
 167                 st->n_bits += src->n_bits;
 168                 st->n_bits_min += var_size ? 0 : src->n_bits;
 169         }
 170         st->n_fields = n_fields;
 171         st->var_size = last_field_has_variable_size;
 172
 173         /* Node add to tailq. */
 174         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 175
 176         return 0;
 177 }
 178
 179 static int
 180 struct_build(struct rte_swx_pipeline *p)
 181 {
 182         uint32_t i;
 183
 184         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 185                 struct thread *t = &p->threads[i];
 186
 187                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 188                 CHECK(t->structs, ENOMEM);
 189         }
 190
 191         return 0;
 192 }
 193
 194 static void
 195 struct_build_free(struct rte_swx_pipeline *p)
 196 {
 197         uint32_t i;
 198
 199         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 200                 struct thread *t = &p->threads[i];
 201
 202                 free(t->structs);
 203                 t->structs = NULL;
 204         }
 205 }
 206
 207 static void
 208 struct_free(struct rte_swx_pipeline *p)
 209 {
 210         struct_build_free(p);
 211
 212         /* Struct types. */
 213         for ( ; ; ) {
 214                 struct struct_type *elem;
 215
 216                 elem = TAILQ_FIRST(&p->struct_types);
 217                 if (!elem)
 218                         break;
 219
 220                 TAILQ_REMOVE(&p->struct_types, elem, node);
 221                 free(elem->fields);
 222                 free(elem);
 223         }
 224 }
 225
 226 /*
 227  * Input port.
 228  */
 229 static struct port_in_type *
 230 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 231 {
 232         struct port_in_type *elem;
 233
 234         if (!name)
 235                 return NULL;
 236
 237         TAILQ_FOREACH(elem, &p->port_in_types, node)
 238                 if (strcmp(elem->name, name) == 0)
 239                         return elem;
 240
 241         return NULL;
 242 }
 243
 244 int
 245 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 246                                        const char *name,
 247                                        struct rte_swx_port_in_ops *ops)
 248 {
 249         struct port_in_type *elem;
 250
 251         CHECK(p, EINVAL);
 252         CHECK_NAME(name, EINVAL);
 253         CHECK(ops, EINVAL);
 254         CHECK(ops->create, EINVAL);
 255         CHECK(ops->free, EINVAL);
 256         CHECK(ops->pkt_rx, EINVAL);
 257         CHECK(ops->stats_read, EINVAL);
 258
 259         CHECK(!port_in_type_find(p, name), EEXIST);
 260
 261         /* Node allocation. */
 262         elem = calloc(1, sizeof(struct port_in_type));
 263         CHECK(elem, ENOMEM);
 264
 265         /* Node initialization. */
 266         strcpy(elem->name, name);
 267         memcpy(&elem->ops, ops, sizeof(*ops));
 268
 269         /* Node add to tailq. */
 270         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 271
 272         return 0;
 273 }
 274
 275 static struct port_in *
 276 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 277 {
 278         struct port_in *port;
 279
 280         TAILQ_FOREACH(port, &p->ports_in, node)
 281                 if (port->id == port_id)
 282                         return port;
 283
 284         return NULL;
 285 }
 286
 287 int
 288 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 289                                 uint32_t port_id,
 290                                 const char *port_type_name,
 291                                 void *args)
 292 {
 293         struct port_in_type *type = NULL;
 294         struct port_in *port = NULL;
 295         void *obj = NULL;
 296
 297         CHECK(p, EINVAL);
 298
 299         CHECK(!port_in_find(p, port_id), EINVAL);
 300
 301         CHECK_NAME(port_type_name, EINVAL);
 302         type = port_in_type_find(p, port_type_name);
 303         CHECK(type, EINVAL);
 304
 305         obj = type->ops.create(args);
 306         CHECK(obj, ENODEV);
 307
 308         /* Node allocation. */
 309         port = calloc(1, sizeof(struct port_in));
 310         CHECK(port, ENOMEM);
 311
 312         /* Node initialization. */
 313         port->type = type;
 314         port->obj = obj;
 315         port->id = port_id;
 316
 317         /* Node add to tailq. */
 318         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 319         if (p->n_ports_in < port_id + 1)
 320                 p->n_ports_in = port_id + 1;
 321
 322         return 0;
 323 }
 324
 325 static int
 326 port_in_build(struct rte_swx_pipeline *p)
 327 {
 328         struct port_in *port;
 329         uint32_t i;
 330
 331         CHECK(p->n_ports_in, EINVAL);
 332         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 333
 334         for (i = 0; i < p->n_ports_in; i++)
 335                 CHECK(port_in_find(p, i), EINVAL);
 336
 337         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 338         CHECK(p->in, ENOMEM);
 339
 340         TAILQ_FOREACH(port, &p->ports_in, node) {
 341                 struct port_in_runtime *in = &p->in[port->id];
 342
 343                 in->pkt_rx = port->type->ops.pkt_rx;
 344                 in->obj = port->obj;
 345         }
 346
 347         return 0;
 348 }
 349
 350 static void
 351 port_in_build_free(struct rte_swx_pipeline *p)
 352 {
 353         free(p->in);
 354         p->in = NULL;
 355 }
 356
 357 static void
 358 port_in_free(struct rte_swx_pipeline *p)
 359 {
 360         port_in_build_free(p);
 361
 362         /* Input ports. */
 363         for ( ; ; ) {
 364                 struct port_in *port;
 365
 366                 port = TAILQ_FIRST(&p->ports_in);
 367                 if (!port)
 368                         break;
 369
 370                 TAILQ_REMOVE(&p->ports_in, port, node);
 371                 port->type->ops.free(port->obj);
 372                 free(port);
 373         }
 374
 375         /* Input port types. */
 376         for ( ; ; ) {
 377                 struct port_in_type *elem;
 378
 379                 elem = TAILQ_FIRST(&p->port_in_types);
 380                 if (!elem)
 381                         break;
 382
 383                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 384                 free(elem);
 385         }
 386 }
 387
 388 /*
 389  * Output port.
 390  */
 391 static struct port_out_type *
 392 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 393 {
 394         struct port_out_type *elem;
 395
 396         if (!name)
 397                 return NULL;
 398
 399         TAILQ_FOREACH(elem, &p->port_out_types, node)
 400                 if (!strcmp(elem->name, name))
 401                         return elem;
 402
 403         return NULL;
 404 }
 405
 406 int
 407 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 408                                         const char *name,
 409                                         struct rte_swx_port_out_ops *ops)
 410 {
 411         struct port_out_type *elem;
 412
 413         CHECK(p, EINVAL);
 414         CHECK_NAME(name, EINVAL);
 415         CHECK(ops, EINVAL);
 416         CHECK(ops->create, EINVAL);
 417         CHECK(ops->free, EINVAL);
 418         CHECK(ops->pkt_tx, EINVAL);
 419         CHECK(ops->stats_read, EINVAL);
 420
 421         CHECK(!port_out_type_find(p, name), EEXIST);
 422
 423         /* Node allocation. */
 424         elem = calloc(1, sizeof(struct port_out_type));
 425         CHECK(elem, ENOMEM);
 426
 427         /* Node initialization. */
 428         strcpy(elem->name, name);
 429         memcpy(&elem->ops, ops, sizeof(*ops));
 430
 431         /* Node add to tailq. */
 432         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 433
 434         return 0;
 435 }
 436
 437 static struct port_out *
 438 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 439 {
 440         struct port_out *port;
 441
 442         TAILQ_FOREACH(port, &p->ports_out, node)
 443                 if (port->id == port_id)
 444                         return port;
 445
 446         return NULL;
 447 }
 448
 449 int
 450 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 451                                  uint32_t port_id,
 452                                  const char *port_type_name,
 453                                  void *args)
 454 {
 455         struct port_out_type *type = NULL;
 456         struct port_out *port = NULL;
 457         void *obj = NULL;
 458
 459         CHECK(p, EINVAL);
 460
 461         CHECK(!port_out_find(p, port_id), EINVAL);
 462
 463         CHECK_NAME(port_type_name, EINVAL);
 464         type = port_out_type_find(p, port_type_name);
 465         CHECK(type, EINVAL);
 466
 467         obj = type->ops.create(args);
 468         CHECK(obj, ENODEV);
 469
 470         /* Node allocation. */
 471         port = calloc(1, sizeof(struct port_out));
 472         CHECK(port, ENOMEM);
 473
 474         /* Node initialization. */
 475         port->type = type;
 476         port->obj = obj;
 477         port->id = port_id;
 478
 479         /* Node add to tailq. */
 480         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 481         if (p->n_ports_out < port_id + 1)
 482                 p->n_ports_out = port_id + 1;
 483
 484         return 0;
 485 }
 486
 487 static int
 488 port_out_build(struct rte_swx_pipeline *p)
 489 {
 490         struct port_out *port;
 491         uint32_t i;
 492
 493         CHECK(p->n_ports_out, EINVAL);
 494
 495         for (i = 0; i < p->n_ports_out; i++)
 496                 CHECK(port_out_find(p, i), EINVAL);
 497
 498         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 499         CHECK(p->out, ENOMEM);
 500
 501         TAILQ_FOREACH(port, &p->ports_out, node) {
 502                 struct port_out_runtime *out = &p->out[port->id];
 503
 504                 out->pkt_tx = port->type->ops.pkt_tx;
 505                 out->flush = port->type->ops.flush;
 506                 out->obj = port->obj;
 507         }
 508
 509         return 0;
 510 }
 511
 512 static void
 513 port_out_build_free(struct rte_swx_pipeline *p)
 514 {
 515         free(p->out);
 516         p->out = NULL;
 517 }
 518
 519 static void
 520 port_out_free(struct rte_swx_pipeline *p)
 521 {
 522         port_out_build_free(p);
 523
 524         /* Output ports. */
 525         for ( ; ; ) {
 526                 struct port_out *port;
 527
 528                 port = TAILQ_FIRST(&p->ports_out);
 529                 if (!port)
 530                         break;
 531
 532                 TAILQ_REMOVE(&p->ports_out, port, node);
 533                 port->type->ops.free(port->obj);
 534                 free(port);
 535         }
 536
 537         /* Output port types. */
 538         for ( ; ; ) {
 539                 struct port_out_type *elem;
 540
 541                 elem = TAILQ_FIRST(&p->port_out_types);
 542                 if (!elem)
 543                         break;
 544
 545                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 546                 free(elem);
 547         }
 548 }
 549
 550 /*
 551  * Extern object.
 552  */
 553 static struct extern_type *
 554 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 555 {
 556         struct extern_type *elem;
 557
 558         TAILQ_FOREACH(elem, &p->extern_types, node)
 559                 if (strcmp(elem->name, name) == 0)
 560                         return elem;
 561
 562         return NULL;
 563 }
 564
 565 static struct extern_type_member_func *
 566 extern_type_member_func_find(struct extern_type *type, const char *name)
 567 {
 568         struct extern_type_member_func *elem;
 569
 570         TAILQ_FOREACH(elem, &type->funcs, node)
 571                 if (strcmp(elem->name, name) == 0)
 572                         return elem;
 573
 574         return NULL;
 575 }
 576
 577 static struct extern_obj *
 578 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 579 {
 580         struct extern_obj *elem;
 581
 582         TAILQ_FOREACH(elem, &p->extern_objs, node)
 583                 if (strcmp(elem->name, name) == 0)
 584                         return elem;
 585
 586         return NULL;
 587 }
 588
 589 static struct extern_type_member_func *
 590 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 591                              const char *name,
 592                              struct extern_obj **obj)
 593 {
 594         struct extern_obj *object;
 595         struct extern_type_member_func *func;
 596         char *object_name, *func_name;
 597
 598         if (name[0] != 'e' || name[1] != '.')
 599                 return NULL;
 600
 601         object_name = strdup(&name[2]);
 602         if (!object_name)
 603                 return NULL;
 604
 605         func_name = strchr(object_name, '.');
 606         if (!func_name) {
 607                 free(object_name);
 608                 return NULL;
 609         }
 610
 611         *func_name = 0;
 612         func_name++;
 613
 614         object = extern_obj_find(p, object_name);
 615         if (!object) {
 616                 free(object_name);
 617                 return NULL;
 618         }
 619
 620         func = extern_type_member_func_find(object->type, func_name);
 621         if (!func) {
 622                 free(object_name);
 623                 return NULL;
 624         }
 625
 626         if (obj)
 627                 *obj = object;
 628
 629         free(object_name);
 630         return func;
 631 }
 632
 633 static struct field *
 634 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 635                                const char *name,
 636                                struct extern_obj **object)
 637 {
 638         struct extern_obj *obj;
 639         struct field *f;
 640         char *obj_name, *field_name;
 641
 642         if ((name[0] != 'e') || (name[1] != '.'))
 643                 return NULL;
 644
 645         obj_name = strdup(&name[2]);
 646         if (!obj_name)
 647                 return NULL;
 648
 649         field_name = strchr(obj_name, '.');
 650         if (!field_name) {
 651                 free(obj_name);
 652                 return NULL;
 653         }
 654
 655         *field_name = 0;
 656         field_name++;
 657
 658         obj = extern_obj_find(p, obj_name);
 659         if (!obj) {
 660                 free(obj_name);
 661                 return NULL;
 662         }
 663
 664         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 665         if (!f) {
 666                 free(obj_name);
 667                 return NULL;
 668         }
 669
 670         if (object)
 671                 *object = obj;
 672
 673         free(obj_name);
 674         return f;
 675 }
 676
 677 int
 678 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 679         const char *name,
 680         const char *mailbox_struct_type_name,
 681         rte_swx_extern_type_constructor_t constructor,
 682         rte_swx_extern_type_destructor_t destructor)
 683 {
 684         struct extern_type *elem;
 685         struct struct_type *mailbox_struct_type;
 686
 687         CHECK(p, EINVAL);
 688
 689         CHECK_NAME(name, EINVAL);
 690         CHECK(!extern_type_find(p, name), EEXIST);
 691
 692         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 693         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 694         CHECK(mailbox_struct_type, EINVAL);
 695         CHECK(!mailbox_struct_type->var_size, EINVAL);
 696
 697         CHECK(constructor, EINVAL);
 698         CHECK(destructor, EINVAL);
 699
 700         /* Node allocation. */
 701         elem = calloc(1, sizeof(struct extern_type));
 702         CHECK(elem, ENOMEM);
 703
 704         /* Node initialization. */
 705         strcpy(elem->name, name);
 706         elem->mailbox_struct_type = mailbox_struct_type;
 707         elem->constructor = constructor;
 708         elem->destructor = destructor;
 709         TAILQ_INIT(&elem->funcs);
 710
 711         /* Node add to tailq. */
 712         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 713
 714         return 0;
 715 }
 716
 717 int
 718 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 719         const char *extern_type_name,
 720         const char *name,
 721         rte_swx_extern_type_member_func_t member_func)
 722 {
 723         struct extern_type *type;
 724         struct extern_type_member_func *type_member;
 725
 726         CHECK(p, EINVAL);
 727
 728         CHECK_NAME(extern_type_name, EINVAL);
 729         type = extern_type_find(p, extern_type_name);
 730         CHECK(type, EINVAL);
 731         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 732
 733         CHECK_NAME(name, EINVAL);
 734         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 735
 736         CHECK(member_func, EINVAL);
 737
 738         /* Node allocation. */
 739         type_member = calloc(1, sizeof(struct extern_type_member_func));
 740         CHECK(type_member, ENOMEM);
 741
 742         /* Node initialization. */
 743         strcpy(type_member->name, name);
 744         type_member->func = member_func;
 745         type_member->id = type->n_funcs;
 746
 747         /* Node add to tailq. */
 748         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 749         type->n_funcs++;
 750
 751         return 0;
 752 }
 753
 754 int
 755 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 756                                       const char *extern_type_name,
 757                                       const char *name,
 758                                       const char *args)
 759 {
 760         struct extern_type *type;
 761         struct extern_obj *obj;
 762         void *obj_handle;
 763
 764         CHECK(p, EINVAL);
 765
 766         CHECK_NAME(extern_type_name, EINVAL);
 767         type = extern_type_find(p, extern_type_name);
 768         CHECK(type, EINVAL);
 769
 770         CHECK_NAME(name, EINVAL);
 771         CHECK(!extern_obj_find(p, name), EEXIST);
 772
 773         /* Node allocation. */
 774         obj = calloc(1, sizeof(struct extern_obj));
 775         CHECK(obj, ENOMEM);
 776
 777         /* Object construction. */
 778         obj_handle = type->constructor(args);
 779         if (!obj_handle) {
 780                 free(obj);
 781                 CHECK(0, ENODEV);
 782         }
 783
 784         /* Node initialization. */
 785         strcpy(obj->name, name);
 786         obj->type = type;
 787         obj->obj = obj_handle;
 788         obj->struct_id = p->n_structs;
 789         obj->id = p->n_extern_objs;
 790
 791         /* Node add to tailq. */
 792         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 793         p->n_extern_objs++;
 794         p->n_structs++;
 795
 796         return 0;
 797 }
 798
 799 static int
 800 extern_obj_build(struct rte_swx_pipeline *p)
 801 {
 802         uint32_t i;
 803
 804         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 805                 struct thread *t = &p->threads[i];
 806                 struct extern_obj *obj;
 807
 808                 t->extern_objs = calloc(p->n_extern_objs,
 809                                         sizeof(struct extern_obj_runtime));
 810                 CHECK(t->extern_objs, ENOMEM);
 811
 812                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 813                         struct extern_obj_runtime *r =
 814                                 &t->extern_objs[obj->id];
 815                         struct extern_type_member_func *func;
 816                         uint32_t mailbox_size =
 817                                 obj->type->mailbox_struct_type->n_bits / 8;
 818
 819                         r->obj = obj->obj;
 820
 821                         r->mailbox = calloc(1, mailbox_size);
 822                         CHECK(r->mailbox, ENOMEM);
 823
 824                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 825                                 r->funcs[func->id] = func->func;
 826
 827                         t->structs[obj->struct_id] = r->mailbox;
 828                 }
 829         }
 830
 831         return 0;
 832 }
 833
 834 static void
 835 extern_obj_build_free(struct rte_swx_pipeline *p)
 836 {
 837         uint32_t i;
 838
 839         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 840                 struct thread *t = &p->threads[i];
 841                 uint32_t j;
 842
 843                 if (!t->extern_objs)
 844                         continue;
 845
 846                 for (j = 0; j < p->n_extern_objs; j++) {
 847                         struct extern_obj_runtime *r = &t->extern_objs[j];
 848
 849                         free(r->mailbox);
 850                 }
 851
 852                 free(t->extern_objs);
 853                 t->extern_objs = NULL;
 854         }
 855 }
 856
 857 static void
 858 extern_obj_free(struct rte_swx_pipeline *p)
 859 {
 860         extern_obj_build_free(p);
 861
 862         /* Extern objects. */
 863         for ( ; ; ) {
 864                 struct extern_obj *elem;
 865
 866                 elem = TAILQ_FIRST(&p->extern_objs);
 867                 if (!elem)
 868                         break;
 869
 870                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 871                 if (elem->obj)
 872                         elem->type->destructor(elem->obj);
 873                 free(elem);
 874         }
 875
 876         /* Extern types. */
 877         for ( ; ; ) {
 878                 struct extern_type *elem;
 879
 880                 elem = TAILQ_FIRST(&p->extern_types);
 881                 if (!elem)
 882                         break;
 883
 884                 TAILQ_REMOVE(&p->extern_types, elem, node);
 885
 886                 for ( ; ; ) {
 887                         struct extern_type_member_func *func;
 888
 889                         func = TAILQ_FIRST(&elem->funcs);
 890                         if (!func)
 891                                 break;
 892
 893                         TAILQ_REMOVE(&elem->funcs, func, node);
 894                         free(func);
 895                 }
 896
 897                 free(elem);
 898         }
 899 }
 900
 901 /*
 902  * Extern function.
 903  */
 904 static struct extern_func *
 905 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 906 {
 907         struct extern_func *elem;
 908
 909         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 910                 if (strcmp(elem->name, name) == 0)
 911                         return elem;
 912
 913         return NULL;
 914 }
 915
 916 static struct extern_func *
 917 extern_func_parse(struct rte_swx_pipeline *p,
 918                   const char *name)
 919 {
 920         if (name[0] != 'f' || name[1] != '.')
 921                 return NULL;
 922
 923         return extern_func_find(p, &name[2]);
 924 }
 925
 926 static struct field *
 927 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 928                                 const char *name,
 929                                 struct extern_func **function)
 930 {
 931         struct extern_func *func;
 932         struct field *f;
 933         char *func_name, *field_name;
 934
 935         if ((name[0] != 'f') || (name[1] != '.'))
 936                 return NULL;
 937
 938         func_name = strdup(&name[2]);
 939         if (!func_name)
 940                 return NULL;
 941
 942         field_name = strchr(func_name, '.');
 943         if (!field_name) {
 944                 free(func_name);
 945                 return NULL;
 946         }
 947
 948         *field_name = 0;
 949         field_name++;
 950
 951         func = extern_func_find(p, func_name);
 952         if (!func) {
 953                 free(func_name);
 954                 return NULL;
 955         }
 956
 957         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 958         if (!f) {
 959                 free(func_name);
 960                 return NULL;
 961         }
 962
 963         if (function)
 964                 *function = func;
 965
 966         free(func_name);
 967         return f;
 968 }
 969
 970 int
 971 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 972                                       const char *name,
 973                                       const char *mailbox_struct_type_name,
 974                                       rte_swx_extern_func_t func)
 975 {
 976         struct extern_func *f;
 977         struct struct_type *mailbox_struct_type;
 978
 979         CHECK(p, EINVAL);
 980
 981         CHECK_NAME(name, EINVAL);
 982         CHECK(!extern_func_find(p, name), EEXIST);
 983
 984         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 985         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 986         CHECK(mailbox_struct_type, EINVAL);
 987         CHECK(!mailbox_struct_type->var_size, EINVAL);
 988
 989         CHECK(func, EINVAL);
 990
 991         /* Node allocation. */
 992         f = calloc(1, sizeof(struct extern_func));
 993         CHECK(func, ENOMEM);
 994
 995         /* Node initialization. */
 996         strcpy(f->name, name);
 997         f->mailbox_struct_type = mailbox_struct_type;
 998         f->func = func;
 999         f->struct_id = p->n_structs;
1000         f->id = p->n_extern_funcs;
1001
1002         /* Node add to tailq. */
1003         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1004         p->n_extern_funcs++;
1005         p->n_structs++;
1006
1007         return 0;
1008 }
1009
1010 static int
1011 extern_func_build(struct rte_swx_pipeline *p)
1012 {
1013         uint32_t i;
1014
1015         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1016                 struct thread *t = &p->threads[i];
1017                 struct extern_func *func;
1018
1019                 /* Memory allocation. */
1020                 t->extern_funcs = calloc(p->n_extern_funcs,
1021                                          sizeof(struct extern_func_runtime));
1022                 CHECK(t->extern_funcs, ENOMEM);
1023
1024                 /* Extern function. */
1025                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1026                         struct extern_func_runtime *r =
1027                                 &t->extern_funcs[func->id];
1028                         uint32_t mailbox_size =
1029                                 func->mailbox_struct_type->n_bits / 8;
1030
1031                         r->func = func->func;
1032
1033                         r->mailbox = calloc(1, mailbox_size);
1034                         CHECK(r->mailbox, ENOMEM);
1035
1036                         t->structs[func->struct_id] = r->mailbox;
1037                 }
1038         }
1039
1040         return 0;
1041 }
1042
1043 static void
1044 extern_func_build_free(struct rte_swx_pipeline *p)
1045 {
1046         uint32_t i;
1047
1048         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1049                 struct thread *t = &p->threads[i];
1050                 uint32_t j;
1051
1052                 if (!t->extern_funcs)
1053                         continue;
1054
1055                 for (j = 0; j < p->n_extern_funcs; j++) {
1056                         struct extern_func_runtime *r = &t->extern_funcs[j];
1057
1058                         free(r->mailbox);
1059                 }
1060
1061                 free(t->extern_funcs);
1062                 t->extern_funcs = NULL;
1063         }
1064 }
1065
1066 static void
1067 extern_func_free(struct rte_swx_pipeline *p)
1068 {
1069         extern_func_build_free(p);
1070
1071         for ( ; ; ) {
1072                 struct extern_func *elem;
1073
1074                 elem = TAILQ_FIRST(&p->extern_funcs);
1075                 if (!elem)
1076                         break;
1077
1078                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1079                 free(elem);
1080         }
1081 }
1082
1083 /*
1084  * Header.
1085  */
1086 static struct header *
1087 header_find(struct rte_swx_pipeline *p, const char *name)
1088 {
1089         struct header *elem;
1090
1091         TAILQ_FOREACH(elem, &p->headers, node)
1092                 if (strcmp(elem->name, name) == 0)
1093                         return elem;
1094
1095         return NULL;
1096 }
1097
1098 static struct header *
1099 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1100 {
1101         struct header *elem;
1102
1103         TAILQ_FOREACH(elem, &p->headers, node)
1104                 if (elem->struct_id == struct_id)
1105                         return elem;
1106
1107         return NULL;
1108 }
1109
1110 static struct header *
1111 header_parse(struct rte_swx_pipeline *p,
1112              const char *name)
1113 {
1114         if (name[0] != 'h' || name[1] != '.')
1115                 return NULL;
1116
1117         return header_find(p, &name[2]);
1118 }
1119
1120 static struct field *
1121 header_field_parse(struct rte_swx_pipeline *p,
1122                    const char *name,
1123                    struct header **header)
1124 {
1125         struct header *h;
1126         struct field *f;
1127         char *header_name, *field_name;
1128
1129         if ((name[0] != 'h') || (name[1] != '.'))
1130                 return NULL;
1131
1132         header_name = strdup(&name[2]);
1133         if (!header_name)
1134                 return NULL;
1135
1136         field_name = strchr(header_name, '.');
1137         if (!field_name) {
1138                 free(header_name);
1139                 return NULL;
1140         }
1141
1142         *field_name = 0;
1143         field_name++;
1144
1145         h = header_find(p, header_name);
1146         if (!h) {
1147                 free(header_name);
1148                 return NULL;
1149         }
1150
1151         f = struct_type_field_find(h->st, field_name);
1152         if (!f) {
1153                 free(header_name);
1154                 return NULL;
1155         }
1156
1157         if (header)
1158                 *header = h;
1159
1160         free(header_name);
1161         return f;
1162 }
1163
1164 int
1165 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1166                                         const char *name,
1167                                         const char *struct_type_name)
1168 {
1169         struct struct_type *st;
1170         struct header *h;
1171         size_t n_headers_max;
1172
1173         CHECK(p, EINVAL);
1174         CHECK_NAME(name, EINVAL);
1175         CHECK_NAME(struct_type_name, EINVAL);
1176
1177         CHECK(!header_find(p, name), EEXIST);
1178
1179         st = struct_type_find(p, struct_type_name);
1180         CHECK(st, EINVAL);
1181
1182         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1183         CHECK(p->n_headers < n_headers_max, ENOSPC);
1184
1185         /* Node allocation. */
1186         h = calloc(1, sizeof(struct header));
1187         CHECK(h, ENOMEM);
1188
1189         /* Node initialization. */
1190         strcpy(h->name, name);
1191         h->st = st;
1192         h->struct_id = p->n_structs;
1193         h->id = p->n_headers;
1194
1195         /* Node add to tailq. */
1196         TAILQ_INSERT_TAIL(&p->headers, h, node);
1197         p->n_headers++;
1198         p->n_structs++;
1199
1200         return 0;
1201 }
1202
1203 static int
1204 header_build(struct rte_swx_pipeline *p)
1205 {
1206         struct header *h;
1207         uint32_t n_bytes = 0, i;
1208
1209         TAILQ_FOREACH(h, &p->headers, node) {
1210                 n_bytes += h->st->n_bits / 8;
1211         }
1212
1213         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1214                 struct thread *t = &p->threads[i];
1215                 uint32_t offset = 0;
1216
1217                 t->headers = calloc(p->n_headers,
1218                                     sizeof(struct header_runtime));
1219                 CHECK(t->headers, ENOMEM);
1220
1221                 t->headers_out = calloc(p->n_headers,
1222                                         sizeof(struct header_out_runtime));
1223                 CHECK(t->headers_out, ENOMEM);
1224
1225                 t->header_storage = calloc(1, n_bytes);
1226                 CHECK(t->header_storage, ENOMEM);
1227
1228                 t->header_out_storage = calloc(1, n_bytes);
1229                 CHECK(t->header_out_storage, ENOMEM);
1230
1231                 TAILQ_FOREACH(h, &p->headers, node) {
1232                         uint8_t *header_storage;
1233                         uint32_t n_bytes =  h->st->n_bits / 8;
1234
1235                         header_storage = &t->header_storage[offset];
1236                         offset += n_bytes;
1237
1238                         t->headers[h->id].ptr0 = header_storage;
1239                         t->headers[h->id].n_bytes = n_bytes;
1240
1241                         t->structs[h->struct_id] = header_storage;
1242                 }
1243         }
1244
1245         return 0;
1246 }
1247
1248 static void
1249 header_build_free(struct rte_swx_pipeline *p)
1250 {
1251         uint32_t i;
1252
1253         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1254                 struct thread *t = &p->threads[i];
1255
1256                 free(t->headers_out);
1257                 t->headers_out = NULL;
1258
1259                 free(t->headers);
1260                 t->headers = NULL;
1261
1262                 free(t->header_out_storage);
1263                 t->header_out_storage = NULL;
1264
1265                 free(t->header_storage);
1266                 t->header_storage = NULL;
1267         }
1268 }
1269
1270 static void
1271 header_free(struct rte_swx_pipeline *p)
1272 {
1273         header_build_free(p);
1274
1275         for ( ; ; ) {
1276                 struct header *elem;
1277
1278                 elem = TAILQ_FIRST(&p->headers);
1279                 if (!elem)
1280                         break;
1281
1282                 TAILQ_REMOVE(&p->headers, elem, node);
1283                 free(elem);
1284         }
1285 }
1286
1287 /*
1288  * Meta-data.
1289  */
1290 static struct field *
1291 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1292 {
1293         if (!p->metadata_st)
1294                 return NULL;
1295
1296         if (name[0] != 'm' || name[1] != '.')
1297                 return NULL;
1298
1299         return struct_type_field_find(p->metadata_st, &name[2]);
1300 }
1301
1302 int
1303 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1304                                           const char *struct_type_name)
1305 {
1306         struct struct_type *st = NULL;
1307
1308         CHECK(p, EINVAL);
1309
1310         CHECK_NAME(struct_type_name, EINVAL);
1311         st  = struct_type_find(p, struct_type_name);
1312         CHECK(st, EINVAL);
1313         CHECK(!st->var_size, EINVAL);
1314         CHECK(!p->metadata_st, EINVAL);
1315
1316         p->metadata_st = st;
1317         p->metadata_struct_id = p->n_structs;
1318
1319         p->n_structs++;
1320
1321         return 0;
1322 }
1323
1324 static int
1325 metadata_build(struct rte_swx_pipeline *p)
1326 {
1327         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1328         uint32_t i;
1329
1330         /* Thread-level initialization. */
1331         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1332                 struct thread *t = &p->threads[i];
1333                 uint8_t *metadata;
1334
1335                 metadata = calloc(1, n_bytes);
1336                 CHECK(metadata, ENOMEM);
1337
1338                 t->metadata = metadata;
1339                 t->structs[p->metadata_struct_id] = metadata;
1340         }
1341
1342         return 0;
1343 }
1344
1345 static void
1346 metadata_build_free(struct rte_swx_pipeline *p)
1347 {
1348         uint32_t i;
1349
1350         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1351                 struct thread *t = &p->threads[i];
1352
1353                 free(t->metadata);
1354                 t->metadata = NULL;
1355         }
1356 }
1357
1358 static void
1359 metadata_free(struct rte_swx_pipeline *p)
1360 {
1361         metadata_build_free(p);
1362 }
1363
1364 /*
1365  * Instruction.
1366  */
1367 static int
1368 instruction_is_tx(enum instruction_type type)
1369 {
1370         switch (type) {
1371         case INSTR_TX:
1372         case INSTR_TX_I:
1373         case INSTR_DROP:
1374                 return 1;
1375
1376         default:
1377                 return 0;
1378         }
1379 }
1380
1381 static int
1382 instruction_does_tx(struct instruction *instr)
1383 {
1384         switch (instr->type) {
1385         case INSTR_TX:
1386         case INSTR_TX_I:
1387         case INSTR_DROP:
1388         case INSTR_HDR_EMIT_TX:
1389         case INSTR_HDR_EMIT2_TX:
1390         case INSTR_HDR_EMIT3_TX:
1391         case INSTR_HDR_EMIT4_TX:
1392         case INSTR_HDR_EMIT5_TX:
1393         case INSTR_HDR_EMIT6_TX:
1394         case INSTR_HDR_EMIT7_TX:
1395         case INSTR_HDR_EMIT8_TX:
1396                 return 1;
1397         default:
1398                 return 0;
1399         }
1400 }
1401
1402 static int
1403 instruction_is_jmp(struct instruction *instr)
1404 {
1405         switch (instr->type) {
1406         case INSTR_JMP:
1407         case INSTR_JMP_VALID:
1408         case INSTR_JMP_INVALID:
1409         case INSTR_JMP_HIT:
1410         case INSTR_JMP_MISS:
1411         case INSTR_JMP_ACTION_HIT:
1412         case INSTR_JMP_ACTION_MISS:
1413         case INSTR_JMP_EQ:
1414         case INSTR_JMP_EQ_MH:
1415         case INSTR_JMP_EQ_HM:
1416         case INSTR_JMP_EQ_HH:
1417         case INSTR_JMP_EQ_I:
1418         case INSTR_JMP_NEQ:
1419         case INSTR_JMP_NEQ_MH:
1420         case INSTR_JMP_NEQ_HM:
1421         case INSTR_JMP_NEQ_HH:
1422         case INSTR_JMP_NEQ_I:
1423         case INSTR_JMP_LT:
1424         case INSTR_JMP_LT_MH:
1425         case INSTR_JMP_LT_HM:
1426         case INSTR_JMP_LT_HH:
1427         case INSTR_JMP_LT_MI:
1428         case INSTR_JMP_LT_HI:
1429         case INSTR_JMP_GT:
1430         case INSTR_JMP_GT_MH:
1431         case INSTR_JMP_GT_HM:
1432         case INSTR_JMP_GT_HH:
1433         case INSTR_JMP_GT_MI:
1434         case INSTR_JMP_GT_HI:
1435                 return 1;
1436
1437         default:
1438                 return 0;
1439         }
1440 }
1441
1442 static int
1443 instruction_does_thread_yield(struct instruction *instr)
1444 {
1445         switch (instr->type) {
1446         case INSTR_RX:
1447         case INSTR_TABLE:
1448         case INSTR_TABLE_AF:
1449         case INSTR_SELECTOR:
1450         case INSTR_LEARNER:
1451         case INSTR_LEARNER_AF:
1452         case INSTR_EXTERN_OBJ:
1453         case INSTR_EXTERN_FUNC:
1454                 return 1;
1455         default:
1456                 return 0;
1457         }
1458 }
1459
1460 static struct field *
1461 action_field_parse(struct action *action, const char *name);
1462
1463 static struct field *
1464 struct_field_parse(struct rte_swx_pipeline *p,
1465                    struct action *action,
1466                    const char *name,
1467                    uint32_t *struct_id)
1468 {
1469         struct field *f;
1470
1471         switch (name[0]) {
1472         case 'h':
1473         {
1474                 struct header *header;
1475
1476                 f = header_field_parse(p, name, &header);
1477                 if (!f)
1478                         return NULL;
1479
1480                 *struct_id = header->struct_id;
1481                 return f;
1482         }
1483
1484         case 'm':
1485         {
1486                 f = metadata_field_parse(p, name);
1487                 if (!f)
1488                         return NULL;
1489
1490                 *struct_id = p->metadata_struct_id;
1491                 return f;
1492         }
1493
1494         case 't':
1495         {
1496                 if (!action)
1497                         return NULL;
1498
1499                 f = action_field_parse(action, name);
1500                 if (!f)
1501                         return NULL;
1502
1503                 *struct_id = 0;
1504                 return f;
1505         }
1506
1507         case 'e':
1508         {
1509                 struct extern_obj *obj;
1510
1511                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1512                 if (!f)
1513                         return NULL;
1514
1515                 *struct_id = obj->struct_id;
1516                 return f;
1517         }
1518
1519         case 'f':
1520         {
1521                 struct extern_func *func;
1522
1523                 f = extern_func_mailbox_field_parse(p, name, &func);
1524                 if (!f)
1525                         return NULL;
1526
1527                 *struct_id = func->struct_id;
1528                 return f;
1529         }
1530
1531         default:
1532                 return NULL;
1533         }
1534 }
1535
1536 /*
1537  * rx.
1538  */
1539 static int
1540 instr_rx_translate(struct rte_swx_pipeline *p,
1541                    struct action *action,
1542                    char **tokens,
1543                    int n_tokens,
1544                    struct instruction *instr,
1545                    struct instruction_data *data __rte_unused)
1546 {
1547         struct field *f;
1548
1549         CHECK(!action, EINVAL);
1550         CHECK(n_tokens == 2, EINVAL);
1551
1552         f = metadata_field_parse(p, tokens[1]);
1553         CHECK(f, EINVAL);
1554
1555         instr->type = INSTR_RX;
1556         instr->io.io.offset = f->offset / 8;
1557         instr->io.io.n_bits = f->n_bits;
1558         return 0;
1559 }
1560
1561 /*
1562  * tx.
1563  */
1564 static int
1565 instr_tx_translate(struct rte_swx_pipeline *p,
1566                    struct action *action __rte_unused,
1567                    char **tokens,
1568                    int n_tokens,
1569                    struct instruction *instr,
1570                    struct instruction_data *data __rte_unused)
1571 {
1572         char *port = tokens[1];
1573         struct field *f;
1574         uint32_t port_val;
1575
1576         CHECK(n_tokens == 2, EINVAL);
1577
1578         f = metadata_field_parse(p, port);
1579         if (f) {
1580                 instr->type = INSTR_TX;
1581                 instr->io.io.offset = f->offset / 8;
1582                 instr->io.io.n_bits = f->n_bits;
1583                 return 0;
1584         }
1585
1586         /* TX_I. */
1587         port_val = strtoul(port, &port, 0);
1588         CHECK(!port[0], EINVAL);
1589
1590         instr->type = INSTR_TX_I;
1591         instr->io.io.val = port_val;
1592         return 0;
1593 }
1594
1595 static int
1596 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1597                      struct action *action __rte_unused,
1598                      char **tokens __rte_unused,
1599                      int n_tokens,
1600                      struct instruction *instr,
1601                      struct instruction_data *data __rte_unused)
1602 {
1603         CHECK(n_tokens == 1, EINVAL);
1604
1605         /* DROP. */
1606         instr->type = INSTR_DROP;
1607         return 0;
1608 }
1609
1610 static inline void
1611 instr_tx_exec(struct rte_swx_pipeline *p)
1612 {
1613         struct thread *t = &p->threads[p->thread_id];
1614         struct instruction *ip = t->ip;
1615
1616         __instr_tx_exec(p, t, ip);
1617
1618         /* Thread. */
1619         thread_ip_reset(p, t);
1620         instr_rx_exec(p);
1621 }
1622
1623 static inline void
1624 instr_tx_i_exec(struct rte_swx_pipeline *p)
1625 {
1626         struct thread *t = &p->threads[p->thread_id];
1627         struct instruction *ip = t->ip;
1628
1629         __instr_tx_i_exec(p, t, ip);
1630
1631         /* Thread. */
1632         thread_ip_reset(p, t);
1633         instr_rx_exec(p);
1634 }
1635
1636 static inline void
1637 instr_drop_exec(struct rte_swx_pipeline *p)
1638 {
1639         struct thread *t = &p->threads[p->thread_id];
1640         struct instruction *ip = t->ip;
1641
1642         __instr_drop_exec(p, t, ip);
1643
1644         /* Thread. */
1645         thread_ip_reset(p, t);
1646         instr_rx_exec(p);
1647 }
1648
1649 /*
1650  * extract.
1651  */
1652 static int
1653 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1654                             struct action *action,
1655                             char **tokens,
1656                             int n_tokens,
1657                             struct instruction *instr,
1658                             struct instruction_data *data __rte_unused)
1659 {
1660         struct header *h;
1661
1662         CHECK(!action, EINVAL);
1663         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1664
1665         h = header_parse(p, tokens[1]);
1666         CHECK(h, EINVAL);
1667
1668         if (n_tokens == 2) {
1669                 CHECK(!h->st->var_size, EINVAL);
1670
1671                 instr->type = INSTR_HDR_EXTRACT;
1672                 instr->io.hdr.header_id[0] = h->id;
1673                 instr->io.hdr.struct_id[0] = h->struct_id;
1674                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1675         } else {
1676                 struct field *mf;
1677
1678                 CHECK(h->st->var_size, EINVAL);
1679
1680                 mf = metadata_field_parse(p, tokens[2]);
1681                 CHECK(mf, EINVAL);
1682                 CHECK(!mf->var_size, EINVAL);
1683
1684                 instr->type = INSTR_HDR_EXTRACT_M;
1685                 instr->io.io.offset = mf->offset / 8;
1686                 instr->io.io.n_bits = mf->n_bits;
1687                 instr->io.hdr.header_id[0] = h->id;
1688                 instr->io.hdr.struct_id[0] = h->struct_id;
1689                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1690         }
1691
1692         return 0;
1693 }
1694
1695 static int
1696 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1697                               struct action *action,
1698                               char **tokens,
1699                               int n_tokens,
1700                               struct instruction *instr,
1701                               struct instruction_data *data __rte_unused)
1702 {
1703         struct header *h;
1704
1705         CHECK(!action, EINVAL);
1706         CHECK(n_tokens == 2, EINVAL);
1707
1708         h = header_parse(p, tokens[1]);
1709         CHECK(h, EINVAL);
1710         CHECK(!h->st->var_size, EINVAL);
1711
1712         instr->type = INSTR_HDR_LOOKAHEAD;
1713         instr->io.hdr.header_id[0] = h->id;
1714         instr->io.hdr.struct_id[0] = h->struct_id;
1715         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1716
1717         return 0;
1718 }
1719
1720 static inline void
1721 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1722 {
1723         struct thread *t = &p->threads[p->thread_id];
1724         struct instruction *ip = t->ip;
1725
1726         __instr_hdr_extract_exec(p, t, ip);
1727
1728         /* Thread. */
1729         thread_ip_inc(p);
1730 }
1731
1732 static inline void
1733 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1734 {
1735         struct thread *t = &p->threads[p->thread_id];
1736         struct instruction *ip = t->ip;
1737
1738         __instr_hdr_extract2_exec(p, t, ip);
1739
1740         /* Thread. */
1741         thread_ip_inc(p);
1742 }
1743
1744 static inline void
1745 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1746 {
1747         struct thread *t = &p->threads[p->thread_id];
1748         struct instruction *ip = t->ip;
1749
1750         __instr_hdr_extract3_exec(p, t, ip);
1751
1752         /* Thread. */
1753         thread_ip_inc(p);
1754 }
1755
1756 static inline void
1757 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1758 {
1759         struct thread *t = &p->threads[p->thread_id];
1760         struct instruction *ip = t->ip;
1761
1762         __instr_hdr_extract4_exec(p, t, ip);
1763
1764         /* Thread. */
1765         thread_ip_inc(p);
1766 }
1767
1768 static inline void
1769 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1770 {
1771         struct thread *t = &p->threads[p->thread_id];
1772         struct instruction *ip = t->ip;
1773
1774         __instr_hdr_extract5_exec(p, t, ip);
1775
1776         /* Thread. */
1777         thread_ip_inc(p);
1778 }
1779
1780 static inline void
1781 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1782 {
1783         struct thread *t = &p->threads[p->thread_id];
1784         struct instruction *ip = t->ip;
1785
1786         __instr_hdr_extract6_exec(p, t, ip);
1787
1788         /* Thread. */
1789         thread_ip_inc(p);
1790 }
1791
1792 static inline void
1793 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1794 {
1795         struct thread *t = &p->threads[p->thread_id];
1796         struct instruction *ip = t->ip;
1797
1798         __instr_hdr_extract7_exec(p, t, ip);
1799
1800         /* Thread. */
1801         thread_ip_inc(p);
1802 }
1803
1804 static inline void
1805 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1806 {
1807         struct thread *t = &p->threads[p->thread_id];
1808         struct instruction *ip = t->ip;
1809
1810         __instr_hdr_extract8_exec(p, t, ip);
1811
1812         /* Thread. */
1813         thread_ip_inc(p);
1814 }
1815
1816 static inline void
1817 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1818 {
1819         struct thread *t = &p->threads[p->thread_id];
1820         struct instruction *ip = t->ip;
1821
1822         __instr_hdr_extract_m_exec(p, t, ip);
1823
1824         /* Thread. */
1825         thread_ip_inc(p);
1826 }
1827
1828 static inline void
1829 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1830 {
1831         struct thread *t = &p->threads[p->thread_id];
1832         struct instruction *ip = t->ip;
1833
1834         __instr_hdr_lookahead_exec(p, t, ip);
1835
1836         /* Thread. */
1837         thread_ip_inc(p);
1838 }
1839
1840 /*
1841  * emit.
1842  */
1843 static int
1844 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1845                          struct action *action __rte_unused,
1846                          char **tokens,
1847                          int n_tokens,
1848                          struct instruction *instr,
1849                          struct instruction_data *data __rte_unused)
1850 {
1851         struct header *h;
1852
1853         CHECK(n_tokens == 2, EINVAL);
1854
1855         h = header_parse(p, tokens[1]);
1856         CHECK(h, EINVAL);
1857
1858         instr->type = INSTR_HDR_EMIT;
1859         instr->io.hdr.header_id[0] = h->id;
1860         instr->io.hdr.struct_id[0] = h->struct_id;
1861         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1862         return 0;
1863 }
1864
1865 static inline void
1866 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1867 {
1868         struct thread *t = &p->threads[p->thread_id];
1869         struct instruction *ip = t->ip;
1870
1871         __instr_hdr_emit_exec(p, t, ip);
1872
1873         /* Thread. */
1874         thread_ip_inc(p);
1875 }
1876
1877 static inline void
1878 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1879 {
1880         struct thread *t = &p->threads[p->thread_id];
1881         struct instruction *ip = t->ip;
1882
1883         __instr_hdr_emit_tx_exec(p, t, ip);
1884
1885         /* Thread. */
1886         thread_ip_reset(p, t);
1887         instr_rx_exec(p);
1888 }
1889
1890 static inline void
1891 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1892 {
1893         struct thread *t = &p->threads[p->thread_id];
1894         struct instruction *ip = t->ip;
1895
1896         __instr_hdr_emit2_tx_exec(p, t, ip);
1897
1898         /* Thread. */
1899         thread_ip_reset(p, t);
1900         instr_rx_exec(p);
1901 }
1902
1903 static inline void
1904 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1905 {
1906         struct thread *t = &p->threads[p->thread_id];
1907         struct instruction *ip = t->ip;
1908
1909         __instr_hdr_emit3_tx_exec(p, t, ip);
1910
1911         /* Thread. */
1912         thread_ip_reset(p, t);
1913         instr_rx_exec(p);
1914 }
1915
1916 static inline void
1917 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1918 {
1919         struct thread *t = &p->threads[p->thread_id];
1920         struct instruction *ip = t->ip;
1921
1922         __instr_hdr_emit4_tx_exec(p, t, ip);
1923
1924         /* Thread. */
1925         thread_ip_reset(p, t);
1926         instr_rx_exec(p);
1927 }
1928
1929 static inline void
1930 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1931 {
1932         struct thread *t = &p->threads[p->thread_id];
1933         struct instruction *ip = t->ip;
1934
1935         __instr_hdr_emit5_tx_exec(p, t, ip);
1936
1937         /* Thread. */
1938         thread_ip_reset(p, t);
1939         instr_rx_exec(p);
1940 }
1941
1942 static inline void
1943 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1944 {
1945         struct thread *t = &p->threads[p->thread_id];
1946         struct instruction *ip = t->ip;
1947
1948         __instr_hdr_emit6_tx_exec(p, t, ip);
1949
1950         /* Thread. */
1951         thread_ip_reset(p, t);
1952         instr_rx_exec(p);
1953 }
1954
1955 static inline void
1956 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1957 {
1958         struct thread *t = &p->threads[p->thread_id];
1959         struct instruction *ip = t->ip;
1960
1961         __instr_hdr_emit7_tx_exec(p, t, ip);
1962
1963         /* Thread. */
1964         thread_ip_reset(p, t);
1965         instr_rx_exec(p);
1966 }
1967
1968 static inline void
1969 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1970 {
1971         struct thread *t = &p->threads[p->thread_id];
1972         struct instruction *ip = t->ip;
1973
1974         __instr_hdr_emit8_tx_exec(p, t, ip);
1975
1976         /* Thread. */
1977         thread_ip_reset(p, t);
1978         instr_rx_exec(p);
1979 }
1980
1981 /*
1982  * validate.
1983  */
1984 static int
1985 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1986                              struct action *action __rte_unused,
1987                              char **tokens,
1988                              int n_tokens,
1989                              struct instruction *instr,
1990                              struct instruction_data *data __rte_unused)
1991 {
1992         struct header *h;
1993
1994         CHECK(n_tokens == 2, EINVAL);
1995
1996         h = header_parse(p, tokens[1]);
1997         CHECK(h, EINVAL);
1998
1999         instr->type = INSTR_HDR_VALIDATE;
2000         instr->valid.header_id = h->id;
2001         return 0;
2002 }
2003
2004 static inline void
2005 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2006 {
2007         struct thread *t = &p->threads[p->thread_id];
2008         struct instruction *ip = t->ip;
2009
2010         __instr_hdr_validate_exec(p, t, ip);
2011
2012         /* Thread. */
2013         thread_ip_inc(p);
2014 }
2015
2016 /*
2017  * invalidate.
2018  */
2019 static int
2020 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2021                                struct action *action __rte_unused,
2022                                char **tokens,
2023                                int n_tokens,
2024                                struct instruction *instr,
2025                                struct instruction_data *data __rte_unused)
2026 {
2027         struct header *h;
2028
2029         CHECK(n_tokens == 2, EINVAL);
2030
2031         h = header_parse(p, tokens[1]);
2032         CHECK(h, EINVAL);
2033
2034         instr->type = INSTR_HDR_INVALIDATE;
2035         instr->valid.header_id = h->id;
2036         return 0;
2037 }
2038
2039 static inline void
2040 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2041 {
2042         struct thread *t = &p->threads[p->thread_id];
2043         struct instruction *ip = t->ip;
2044
2045         __instr_hdr_invalidate_exec(p, t, ip);
2046
2047         /* Thread. */
2048         thread_ip_inc(p);
2049 }
2050
2051 /*
2052  * table.
2053  */
2054 static struct table *
2055 table_find(struct rte_swx_pipeline *p, const char *name);
2056
2057 static struct selector *
2058 selector_find(struct rte_swx_pipeline *p, const char *name);
2059
2060 static struct learner *
2061 learner_find(struct rte_swx_pipeline *p, const char *name);
2062
2063 static int
2064 instr_table_translate(struct rte_swx_pipeline *p,
2065                       struct action *action,
2066                       char **tokens,
2067                       int n_tokens,
2068                       struct instruction *instr,
2069                       struct instruction_data *data __rte_unused)
2070 {
2071         struct table *t;
2072         struct selector *s;
2073         struct learner *l;
2074
2075         CHECK(!action, EINVAL);
2076         CHECK(n_tokens == 2, EINVAL);
2077
2078         t = table_find(p, tokens[1]);
2079         if (t) {
2080                 instr->type = INSTR_TABLE;
2081                 instr->table.table_id = t->id;
2082                 return 0;
2083         }
2084
2085         s = selector_find(p, tokens[1]);
2086         if (s) {
2087                 instr->type = INSTR_SELECTOR;
2088                 instr->table.table_id = s->id;
2089                 return 0;
2090         }
2091
2092         l = learner_find(p, tokens[1]);
2093         if (l) {
2094                 instr->type = INSTR_LEARNER;
2095                 instr->table.table_id = l->id;
2096                 return 0;
2097         }
2098
2099         CHECK(0, EINVAL);
2100 }
2101
2102 static inline void
2103 instr_table_exec(struct rte_swx_pipeline *p)
2104 {
2105         struct thread *t = &p->threads[p->thread_id];
2106         struct instruction *ip = t->ip;
2107         uint32_t table_id = ip->table.table_id;
2108         struct rte_swx_table_state *ts = &t->table_state[table_id];
2109         struct table_runtime *table = &t->tables[table_id];
2110         struct table_statistics *stats = &p->table_stats[table_id];
2111         uint64_t action_id, n_pkts_hit, n_pkts_action;
2112         uint8_t *action_data;
2113         int done, hit;
2114
2115         /* Table. */
2116         done = table->func(ts->obj,
2117                            table->mailbox,
2118                            table->key,
2119                            &action_id,
2120                            &action_data,
2121                            &hit);
2122         if (!done) {
2123                 /* Thread. */
2124                 TRACE("[Thread %2u] table %u (not finalized)\n",
2125                       p->thread_id,
2126                       table_id);
2127
2128                 thread_yield(p);
2129                 return;
2130         }
2131
2132         action_id = hit ? action_id : ts->default_action_id;
2133         action_data = hit ? action_data : ts->default_action_data;
2134         n_pkts_hit = stats->n_pkts_hit[hit];
2135         n_pkts_action = stats->n_pkts_action[action_id];
2136
2137         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2138               p->thread_id,
2139               table_id,
2140               hit ? "hit" : "miss",
2141               (uint32_t)action_id);
2142
2143         t->action_id = action_id;
2144         t->structs[0] = action_data;
2145         t->hit = hit;
2146         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2147         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2148
2149         /* Thread. */
2150         thread_ip_action_call(p, t, action_id);
2151 }
2152
2153 static inline void
2154 instr_table_af_exec(struct rte_swx_pipeline *p)
2155 {
2156         struct thread *t = &p->threads[p->thread_id];
2157         struct instruction *ip = t->ip;
2158         uint32_t table_id = ip->table.table_id;
2159         struct rte_swx_table_state *ts = &t->table_state[table_id];
2160         struct table_runtime *table = &t->tables[table_id];
2161         struct table_statistics *stats = &p->table_stats[table_id];
2162         uint64_t action_id, n_pkts_hit, n_pkts_action;
2163         uint8_t *action_data;
2164         action_func_t action_func;
2165         int done, hit;
2166
2167         /* Table. */
2168         done = table->func(ts->obj,
2169                            table->mailbox,
2170                            table->key,
2171                            &action_id,
2172                            &action_data,
2173                            &hit);
2174         if (!done) {
2175                 /* Thread. */
2176                 TRACE("[Thread %2u] table %u (not finalized)\n",
2177                       p->thread_id,
2178                       table_id);
2179
2180                 thread_yield(p);
2181                 return;
2182         }
2183
2184         action_id = hit ? action_id : ts->default_action_id;
2185         action_data = hit ? action_data : ts->default_action_data;
2186         action_func = p->action_funcs[action_id];
2187         n_pkts_hit = stats->n_pkts_hit[hit];
2188         n_pkts_action = stats->n_pkts_action[action_id];
2189
2190         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2191               p->thread_id,
2192               table_id,
2193               hit ? "hit" : "miss",
2194               (uint32_t)action_id);
2195
2196         t->action_id = action_id;
2197         t->structs[0] = action_data;
2198         t->hit = hit;
2199         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2200         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2201
2202         /* Thread. */
2203         thread_ip_inc(p);
2204
2205         /* Action. */
2206         action_func(p);
2207 }
2208
2209 static inline void
2210 instr_selector_exec(struct rte_swx_pipeline *p)
2211 {
2212         struct thread *t = &p->threads[p->thread_id];
2213         struct instruction *ip = t->ip;
2214         uint32_t selector_id = ip->table.table_id;
2215         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2216         struct selector_runtime *selector = &t->selectors[selector_id];
2217         struct selector_statistics *stats = &p->selector_stats[selector_id];
2218         uint64_t n_pkts = stats->n_pkts;
2219         int done;
2220
2221         /* Table. */
2222         done = rte_swx_table_selector_select(ts->obj,
2223                            selector->mailbox,
2224                            selector->group_id_buffer,
2225                            selector->selector_buffer,
2226                            selector->member_id_buffer);
2227         if (!done) {
2228                 /* Thread. */
2229                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2230                       p->thread_id,
2231                       selector_id);
2232
2233                 thread_yield(p);
2234                 return;
2235         }
2236
2237
2238         TRACE("[Thread %2u] selector %u\n",
2239               p->thread_id,
2240               selector_id);
2241
2242         stats->n_pkts = n_pkts + 1;
2243
2244         /* Thread. */
2245         thread_ip_inc(p);
2246 }
2247
2248 static inline void
2249 instr_learner_exec(struct rte_swx_pipeline *p)
2250 {
2251         struct thread *t = &p->threads[p->thread_id];
2252         struct instruction *ip = t->ip;
2253         uint32_t learner_id = ip->table.table_id;
2254         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2255                 p->n_selectors + learner_id];
2256         struct learner_runtime *l = &t->learners[learner_id];
2257         struct learner_statistics *stats = &p->learner_stats[learner_id];
2258         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2259         uint8_t *action_data;
2260         int done, hit;
2261
2262         /* Table. */
2263         time = rte_get_tsc_cycles();
2264
2265         done = rte_swx_table_learner_lookup(ts->obj,
2266                                             l->mailbox,
2267                                             time,
2268                                             l->key,
2269                                             &action_id,
2270                                             &action_data,
2271                                             &hit);
2272         if (!done) {
2273                 /* Thread. */
2274                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2275                       p->thread_id,
2276                       learner_id);
2277
2278                 thread_yield(p);
2279                 return;
2280         }
2281
2282         action_id = hit ? action_id : ts->default_action_id;
2283         action_data = hit ? action_data : ts->default_action_data;
2284         n_pkts_hit = stats->n_pkts_hit[hit];
2285         n_pkts_action = stats->n_pkts_action[action_id];
2286
2287         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2288               p->thread_id,
2289               learner_id,
2290               hit ? "hit" : "miss",
2291               (uint32_t)action_id);
2292
2293         t->action_id = action_id;
2294         t->structs[0] = action_data;
2295         t->hit = hit;
2296         t->learner_id = learner_id;
2297         t->time = time;
2298         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2299         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2300
2301         /* Thread. */
2302         thread_ip_action_call(p, t, action_id);
2303 }
2304
2305 static inline void
2306 instr_learner_af_exec(struct rte_swx_pipeline *p)
2307 {
2308         struct thread *t = &p->threads[p->thread_id];
2309         struct instruction *ip = t->ip;
2310         uint32_t learner_id = ip->table.table_id;
2311         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2312                 p->n_selectors + learner_id];
2313         struct learner_runtime *l = &t->learners[learner_id];
2314         struct learner_statistics *stats = &p->learner_stats[learner_id];
2315         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2316         uint8_t *action_data;
2317         action_func_t action_func;
2318         int done, hit;
2319
2320         /* Table. */
2321         time = rte_get_tsc_cycles();
2322
2323         done = rte_swx_table_learner_lookup(ts->obj,
2324                                             l->mailbox,
2325                                             time,
2326                                             l->key,
2327                                             &action_id,
2328                                             &action_data,
2329                                             &hit);
2330         if (!done) {
2331                 /* Thread. */
2332                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2333                       p->thread_id,
2334                       learner_id);
2335
2336                 thread_yield(p);
2337                 return;
2338         }
2339
2340         action_id = hit ? action_id : ts->default_action_id;
2341         action_data = hit ? action_data : ts->default_action_data;
2342         action_func = p->action_funcs[action_id];
2343         n_pkts_hit = stats->n_pkts_hit[hit];
2344         n_pkts_action = stats->n_pkts_action[action_id];
2345
2346         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2347               p->thread_id,
2348               learner_id,
2349               hit ? "hit" : "miss",
2350               (uint32_t)action_id);
2351
2352         t->action_id = action_id;
2353         t->structs[0] = action_data;
2354         t->hit = hit;
2355         t->learner_id = learner_id;
2356         t->time = time;
2357         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2358         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2359
2360         /* Thread. */
2361         thread_ip_action_call(p, t, action_id);
2362
2363         /* Action */
2364         action_func(p);
2365 }
2366
2367 /*
2368  * learn.
2369  */
2370 static struct action *
2371 action_find(struct rte_swx_pipeline *p, const char *name);
2372
2373 static int
2374 action_has_nbo_args(struct action *a);
2375
2376 static int
2377 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2378
2379 static int
2380 instr_learn_translate(struct rte_swx_pipeline *p,
2381                       struct action *action,
2382                       char **tokens,
2383                       int n_tokens,
2384                       struct instruction *instr,
2385                       struct instruction_data *data __rte_unused)
2386 {
2387         struct action *a;
2388         const char *mf_name;
2389         uint32_t mf_offset = 0;
2390
2391         CHECK(action, EINVAL);
2392         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2393
2394         a = action_find(p, tokens[1]);
2395         CHECK(a, EINVAL);
2396         CHECK(!action_has_nbo_args(a), EINVAL);
2397
2398         mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2399         CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2400
2401         if (mf_name) {
2402                 struct field *mf;
2403
2404                 mf = metadata_field_parse(p, mf_name);
2405                 CHECK(mf, EINVAL);
2406
2407                 mf_offset = mf->offset / 8;
2408         }
2409
2410         instr->type = INSTR_LEARNER_LEARN;
2411         instr->learn.action_id = a->id;
2412         instr->learn.mf_offset = mf_offset;
2413
2414         return 0;
2415 }
2416
2417 static inline void
2418 instr_learn_exec(struct rte_swx_pipeline *p)
2419 {
2420         struct thread *t = &p->threads[p->thread_id];
2421         struct instruction *ip = t->ip;
2422
2423         __instr_learn_exec(p, t, ip);
2424
2425         /* Thread. */
2426         thread_ip_inc(p);
2427 }
2428
2429 /*
2430  * forget.
2431  */
2432 static int
2433 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2434                        struct action *action,
2435                        char **tokens __rte_unused,
2436                        int n_tokens,
2437                        struct instruction *instr,
2438                        struct instruction_data *data __rte_unused)
2439 {
2440         CHECK(action, EINVAL);
2441         CHECK(n_tokens == 1, EINVAL);
2442
2443         instr->type = INSTR_LEARNER_FORGET;
2444
2445         return 0;
2446 }
2447
2448 static inline void
2449 instr_forget_exec(struct rte_swx_pipeline *p)
2450 {
2451         struct thread *t = &p->threads[p->thread_id];
2452         struct instruction *ip = t->ip;
2453
2454         __instr_forget_exec(p, t, ip);
2455
2456         /* Thread. */
2457         thread_ip_inc(p);
2458 }
2459
2460 /*
2461  * extern.
2462  */
2463 static int
2464 instr_extern_translate(struct rte_swx_pipeline *p,
2465                        struct action *action __rte_unused,
2466                        char **tokens,
2467                        int n_tokens,
2468                        struct instruction *instr,
2469                        struct instruction_data *data __rte_unused)
2470 {
2471         char *token = tokens[1];
2472
2473         CHECK(n_tokens == 2, EINVAL);
2474
2475         if (token[0] == 'e') {
2476                 struct extern_obj *obj;
2477                 struct extern_type_member_func *func;
2478
2479                 func = extern_obj_member_func_parse(p, token, &obj);
2480                 CHECK(func, EINVAL);
2481
2482                 instr->type = INSTR_EXTERN_OBJ;
2483                 instr->ext_obj.ext_obj_id = obj->id;
2484                 instr->ext_obj.func_id = func->id;
2485
2486                 return 0;
2487         }
2488
2489         if (token[0] == 'f') {
2490                 struct extern_func *func;
2491
2492                 func = extern_func_parse(p, token);
2493                 CHECK(func, EINVAL);
2494
2495                 instr->type = INSTR_EXTERN_FUNC;
2496                 instr->ext_func.ext_func_id = func->id;
2497
2498                 return 0;
2499         }
2500
2501         CHECK(0, EINVAL);
2502 }
2503
2504 static inline void
2505 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2506 {
2507         struct thread *t = &p->threads[p->thread_id];
2508         struct instruction *ip = t->ip;
2509         uint32_t done;
2510
2511         /* Extern object member function execute. */
2512         done = __instr_extern_obj_exec(p, t, ip);
2513
2514         /* Thread. */
2515         thread_ip_inc_cond(t, done);
2516         thread_yield_cond(p, done ^ 1);
2517 }
2518
2519 static inline void
2520 instr_extern_func_exec(struct rte_swx_pipeline *p)
2521 {
2522         struct thread *t = &p->threads[p->thread_id];
2523         struct instruction *ip = t->ip;
2524         uint32_t done;
2525
2526         /* Extern function execute. */
2527         done = __instr_extern_func_exec(p, t, ip);
2528
2529         /* Thread. */
2530         thread_ip_inc_cond(t, done);
2531         thread_yield_cond(p, done ^ 1);
2532 }
2533
2534 /*
2535  * mov.
2536  */
2537 static int
2538 instr_mov_translate(struct rte_swx_pipeline *p,
2539                     struct action *action,
2540                     char **tokens,
2541                     int n_tokens,
2542                     struct instruction *instr,
2543                     struct instruction_data *data __rte_unused)
2544 {
2545         char *dst = tokens[1], *src = tokens[2];
2546         struct field *fdst, *fsrc;
2547         uint64_t src_val;
2548         uint32_t dst_struct_id = 0, src_struct_id = 0;
2549
2550         CHECK(n_tokens == 3, EINVAL);
2551
2552         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2553         CHECK(fdst, EINVAL);
2554         CHECK(!fdst->var_size, EINVAL);
2555
2556         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2557         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2558         if (fsrc) {
2559                 CHECK(!fsrc->var_size, EINVAL);
2560
2561                 instr->type = INSTR_MOV;
2562                 if (dst[0] != 'h' && src[0] == 'h')
2563                         instr->type = INSTR_MOV_MH;
2564                 if (dst[0] == 'h' && src[0] != 'h')
2565                         instr->type = INSTR_MOV_HM;
2566                 if (dst[0] == 'h' && src[0] == 'h')
2567                         instr->type = INSTR_MOV_HH;
2568
2569                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2570                 instr->mov.dst.n_bits = fdst->n_bits;
2571                 instr->mov.dst.offset = fdst->offset / 8;
2572                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2573                 instr->mov.src.n_bits = fsrc->n_bits;
2574                 instr->mov.src.offset = fsrc->offset / 8;
2575                 return 0;
2576         }
2577
2578         /* MOV_I. */
2579         src_val = strtoull(src, &src, 0);
2580         CHECK(!src[0], EINVAL);
2581
2582         if (dst[0] == 'h')
2583                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2584
2585         instr->type = INSTR_MOV_I;
2586         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2587         instr->mov.dst.n_bits = fdst->n_bits;
2588         instr->mov.dst.offset = fdst->offset / 8;
2589         instr->mov.src_val = src_val;
2590         return 0;
2591 }
2592
2593 static inline void
2594 instr_mov_exec(struct rte_swx_pipeline *p)
2595 {
2596         struct thread *t = &p->threads[p->thread_id];
2597         struct instruction *ip = t->ip;
2598
2599         __instr_mov_exec(p, t, ip);
2600
2601         /* Thread. */
2602         thread_ip_inc(p);
2603 }
2604
2605 static inline void
2606 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2607 {
2608         struct thread *t = &p->threads[p->thread_id];
2609         struct instruction *ip = t->ip;
2610
2611         __instr_mov_mh_exec(p, t, ip);
2612
2613         /* Thread. */
2614         thread_ip_inc(p);
2615 }
2616
2617 static inline void
2618 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2619 {
2620         struct thread *t = &p->threads[p->thread_id];
2621         struct instruction *ip = t->ip;
2622
2623         __instr_mov_hm_exec(p, t, ip);
2624
2625         /* Thread. */
2626         thread_ip_inc(p);
2627 }
2628
2629 static inline void
2630 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2631 {
2632         struct thread *t = &p->threads[p->thread_id];
2633         struct instruction *ip = t->ip;
2634
2635         __instr_mov_hh_exec(p, t, ip);
2636
2637         /* Thread. */
2638         thread_ip_inc(p);
2639 }
2640
2641 static inline void
2642 instr_mov_i_exec(struct rte_swx_pipeline *p)
2643 {
2644         struct thread *t = &p->threads[p->thread_id];
2645         struct instruction *ip = t->ip;
2646
2647         __instr_mov_i_exec(p, t, ip);
2648
2649         /* Thread. */
2650         thread_ip_inc(p);
2651 }
2652
2653 /*
2654  * dma.
2655  */
2656 static inline void
2657 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2658 {
2659         struct thread *t = &p->threads[p->thread_id];
2660         struct instruction *ip = t->ip;
2661
2662         __instr_dma_ht_exec(p, t, ip);
2663
2664         /* Thread. */
2665         thread_ip_inc(p);
2666 }
2667
2668 static inline void
2669 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2670 {
2671         struct thread *t = &p->threads[p->thread_id];
2672         struct instruction *ip = t->ip;
2673
2674         __instr_dma_ht2_exec(p, t, ip);
2675
2676         /* Thread. */
2677         thread_ip_inc(p);
2678 }
2679
2680 static inline void
2681 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2682 {
2683         struct thread *t = &p->threads[p->thread_id];
2684         struct instruction *ip = t->ip;
2685
2686         __instr_dma_ht3_exec(p, t, ip);
2687
2688         /* Thread. */
2689         thread_ip_inc(p);
2690 }
2691
2692 static inline void
2693 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2694 {
2695         struct thread *t = &p->threads[p->thread_id];
2696         struct instruction *ip = t->ip;
2697
2698         __instr_dma_ht4_exec(p, t, ip);
2699
2700         /* Thread. */
2701         thread_ip_inc(p);
2702 }
2703
2704 static inline void
2705 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2706 {
2707         struct thread *t = &p->threads[p->thread_id];
2708         struct instruction *ip = t->ip;
2709
2710         __instr_dma_ht5_exec(p, t, ip);
2711
2712         /* Thread. */
2713         thread_ip_inc(p);
2714 }
2715
2716 static inline void
2717 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2718 {
2719         struct thread *t = &p->threads[p->thread_id];
2720         struct instruction *ip = t->ip;
2721
2722         __instr_dma_ht6_exec(p, t, ip);
2723
2724         /* Thread. */
2725         thread_ip_inc(p);
2726 }
2727
2728 static inline void
2729 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2730 {
2731         struct thread *t = &p->threads[p->thread_id];
2732         struct instruction *ip = t->ip;
2733
2734         __instr_dma_ht7_exec(p, t, ip);
2735
2736         /* Thread. */
2737         thread_ip_inc(p);
2738 }
2739
2740 static inline void
2741 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2742 {
2743         struct thread *t = &p->threads[p->thread_id];
2744         struct instruction *ip = t->ip;
2745
2746         __instr_dma_ht8_exec(p, t, ip);
2747
2748         /* Thread. */
2749         thread_ip_inc(p);
2750 }
2751
2752 /*
2753  * alu.
2754  */
2755 static int
2756 instr_alu_add_translate(struct rte_swx_pipeline *p,
2757                         struct action *action,
2758                         char **tokens,
2759                         int n_tokens,
2760                         struct instruction *instr,
2761                         struct instruction_data *data __rte_unused)
2762 {
2763         char *dst = tokens[1], *src = tokens[2];
2764         struct field *fdst, *fsrc;
2765         uint64_t src_val;
2766         uint32_t dst_struct_id = 0, src_struct_id = 0;
2767
2768         CHECK(n_tokens == 3, EINVAL);
2769
2770         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2771         CHECK(fdst, EINVAL);
2772         CHECK(!fdst->var_size, EINVAL);
2773
2774         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2775         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2776         if (fsrc) {
2777                 CHECK(!fsrc->var_size, EINVAL);
2778
2779                 instr->type = INSTR_ALU_ADD;
2780                 if (dst[0] == 'h' && src[0] != 'h')
2781                         instr->type = INSTR_ALU_ADD_HM;
2782                 if (dst[0] != 'h' && src[0] == 'h')
2783                         instr->type = INSTR_ALU_ADD_MH;
2784                 if (dst[0] == 'h' && src[0] == 'h')
2785                         instr->type = INSTR_ALU_ADD_HH;
2786
2787                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2788                 instr->alu.dst.n_bits = fdst->n_bits;
2789                 instr->alu.dst.offset = fdst->offset / 8;
2790                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2791                 instr->alu.src.n_bits = fsrc->n_bits;
2792                 instr->alu.src.offset = fsrc->offset / 8;
2793                 return 0;
2794         }
2795
2796         /* ADD_MI, ADD_HI. */
2797         src_val = strtoull(src, &src, 0);
2798         CHECK(!src[0], EINVAL);
2799
2800         instr->type = INSTR_ALU_ADD_MI;
2801         if (dst[0] == 'h')
2802                 instr->type = INSTR_ALU_ADD_HI;
2803
2804         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2805         instr->alu.dst.n_bits = fdst->n_bits;
2806         instr->alu.dst.offset = fdst->offset / 8;
2807         instr->alu.src_val = src_val;
2808         return 0;
2809 }
2810
2811 static int
2812 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2813                         struct action *action,
2814                         char **tokens,
2815                         int n_tokens,
2816                         struct instruction *instr,
2817                         struct instruction_data *data __rte_unused)
2818 {
2819         char *dst = tokens[1], *src = tokens[2];
2820         struct field *fdst, *fsrc;
2821         uint64_t src_val;
2822         uint32_t dst_struct_id = 0, src_struct_id = 0;
2823
2824         CHECK(n_tokens == 3, EINVAL);
2825
2826         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2827         CHECK(fdst, EINVAL);
2828         CHECK(!fdst->var_size, EINVAL);
2829
2830         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2831         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2832         if (fsrc) {
2833                 CHECK(!fsrc->var_size, EINVAL);
2834
2835                 instr->type = INSTR_ALU_SUB;
2836                 if (dst[0] == 'h' && src[0] != 'h')
2837                         instr->type = INSTR_ALU_SUB_HM;
2838                 if (dst[0] != 'h' && src[0] == 'h')
2839                         instr->type = INSTR_ALU_SUB_MH;
2840                 if (dst[0] == 'h' && src[0] == 'h')
2841                         instr->type = INSTR_ALU_SUB_HH;
2842
2843                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2844                 instr->alu.dst.n_bits = fdst->n_bits;
2845                 instr->alu.dst.offset = fdst->offset / 8;
2846                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2847                 instr->alu.src.n_bits = fsrc->n_bits;
2848                 instr->alu.src.offset = fsrc->offset / 8;
2849                 return 0;
2850         }
2851
2852         /* SUB_MI, SUB_HI. */
2853         src_val = strtoull(src, &src, 0);
2854         CHECK(!src[0], EINVAL);
2855
2856         instr->type = INSTR_ALU_SUB_MI;
2857         if (dst[0] == 'h')
2858                 instr->type = INSTR_ALU_SUB_HI;
2859
2860         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2861         instr->alu.dst.n_bits = fdst->n_bits;
2862         instr->alu.dst.offset = fdst->offset / 8;
2863         instr->alu.src_val = src_val;
2864         return 0;
2865 }
2866
2867 static int
2868 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2869                           struct action *action __rte_unused,
2870                           char **tokens,
2871                           int n_tokens,
2872                           struct instruction *instr,
2873                           struct instruction_data *data __rte_unused)
2874 {
2875         char *dst = tokens[1], *src = tokens[2];
2876         struct header *hdst, *hsrc;
2877         struct field *fdst, *fsrc;
2878
2879         CHECK(n_tokens == 3, EINVAL);
2880
2881         fdst = header_field_parse(p, dst, &hdst);
2882         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2883         CHECK(!fdst->var_size, EINVAL);
2884
2885         /* CKADD_FIELD. */
2886         fsrc = header_field_parse(p, src, &hsrc);
2887         if (fsrc) {
2888                 CHECK(!fsrc->var_size, EINVAL);
2889
2890                 instr->type = INSTR_ALU_CKADD_FIELD;
2891                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2892                 instr->alu.dst.n_bits = fdst->n_bits;
2893                 instr->alu.dst.offset = fdst->offset / 8;
2894                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2895                 instr->alu.src.n_bits = fsrc->n_bits;
2896                 instr->alu.src.offset = fsrc->offset / 8;
2897                 return 0;
2898         }
2899
2900         /* CKADD_STRUCT, CKADD_STRUCT20. */
2901         hsrc = header_parse(p, src);
2902         CHECK(hsrc, EINVAL);
2903         CHECK(!hsrc->st->var_size, EINVAL);
2904
2905         instr->type = INSTR_ALU_CKADD_STRUCT;
2906         if ((hsrc->st->n_bits / 8) == 20)
2907                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2908
2909         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2910         instr->alu.dst.n_bits = fdst->n_bits;
2911         instr->alu.dst.offset = fdst->offset / 8;
2912         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2913         instr->alu.src.n_bits = hsrc->st->n_bits;
2914         instr->alu.src.offset = 0; /* Unused. */
2915         return 0;
2916 }
2917
2918 static int
2919 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2920                           struct action *action __rte_unused,
2921                           char **tokens,
2922                           int n_tokens,
2923                           struct instruction *instr,
2924                           struct instruction_data *data __rte_unused)
2925 {
2926         char *dst = tokens[1], *src = tokens[2];
2927         struct header *hdst, *hsrc;
2928         struct field *fdst, *fsrc;
2929
2930         CHECK(n_tokens == 3, EINVAL);
2931
2932         fdst = header_field_parse(p, dst, &hdst);
2933         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2934         CHECK(!fdst->var_size, EINVAL);
2935
2936         fsrc = header_field_parse(p, src, &hsrc);
2937         CHECK(fsrc, EINVAL);
2938         CHECK(!fsrc->var_size, EINVAL);
2939
2940         instr->type = INSTR_ALU_CKSUB_FIELD;
2941         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2942         instr->alu.dst.n_bits = fdst->n_bits;
2943         instr->alu.dst.offset = fdst->offset / 8;
2944         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2945         instr->alu.src.n_bits = fsrc->n_bits;
2946         instr->alu.src.offset = fsrc->offset / 8;
2947         return 0;
2948 }
2949
2950 static int
2951 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2952                         struct action *action,
2953                         char **tokens,
2954                         int n_tokens,
2955                         struct instruction *instr,
2956                         struct instruction_data *data __rte_unused)
2957 {
2958         char *dst = tokens[1], *src = tokens[2];
2959         struct field *fdst, *fsrc;
2960         uint64_t src_val;
2961         uint32_t dst_struct_id = 0, src_struct_id = 0;
2962
2963         CHECK(n_tokens == 3, EINVAL);
2964
2965         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2966         CHECK(fdst, EINVAL);
2967         CHECK(!fdst->var_size, EINVAL);
2968
2969         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2970         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2971         if (fsrc) {
2972                 CHECK(!fsrc->var_size, EINVAL);
2973
2974                 instr->type = INSTR_ALU_SHL;
2975                 if (dst[0] == 'h' && src[0] != 'h')
2976                         instr->type = INSTR_ALU_SHL_HM;
2977                 if (dst[0] != 'h' && src[0] == 'h')
2978                         instr->type = INSTR_ALU_SHL_MH;
2979                 if (dst[0] == 'h' && src[0] == 'h')
2980                         instr->type = INSTR_ALU_SHL_HH;
2981
2982                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2983                 instr->alu.dst.n_bits = fdst->n_bits;
2984                 instr->alu.dst.offset = fdst->offset / 8;
2985                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2986                 instr->alu.src.n_bits = fsrc->n_bits;
2987                 instr->alu.src.offset = fsrc->offset / 8;
2988                 return 0;
2989         }
2990
2991         /* SHL_MI, SHL_HI. */
2992         src_val = strtoull(src, &src, 0);
2993         CHECK(!src[0], EINVAL);
2994
2995         instr->type = INSTR_ALU_SHL_MI;
2996         if (dst[0] == 'h')
2997                 instr->type = INSTR_ALU_SHL_HI;
2998
2999         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3000         instr->alu.dst.n_bits = fdst->n_bits;
3001         instr->alu.dst.offset = fdst->offset / 8;
3002         instr->alu.src_val = src_val;
3003         return 0;
3004 }
3005
3006 static int
3007 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3008                         struct action *action,
3009                         char **tokens,
3010                         int n_tokens,
3011                         struct instruction *instr,
3012                         struct instruction_data *data __rte_unused)
3013 {
3014         char *dst = tokens[1], *src = tokens[2];
3015         struct field *fdst, *fsrc;
3016         uint64_t src_val;
3017         uint32_t dst_struct_id = 0, src_struct_id = 0;
3018
3019         CHECK(n_tokens == 3, EINVAL);
3020
3021         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3022         CHECK(fdst, EINVAL);
3023         CHECK(!fdst->var_size, EINVAL);
3024
3025         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3026         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3027         if (fsrc) {
3028                 CHECK(!fsrc->var_size, EINVAL);
3029
3030                 instr->type = INSTR_ALU_SHR;
3031                 if (dst[0] == 'h' && src[0] != 'h')
3032                         instr->type = INSTR_ALU_SHR_HM;
3033                 if (dst[0] != 'h' && src[0] == 'h')
3034                         instr->type = INSTR_ALU_SHR_MH;
3035                 if (dst[0] == 'h' && src[0] == 'h')
3036                         instr->type = INSTR_ALU_SHR_HH;
3037
3038                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3039                 instr->alu.dst.n_bits = fdst->n_bits;
3040                 instr->alu.dst.offset = fdst->offset / 8;
3041                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3042                 instr->alu.src.n_bits = fsrc->n_bits;
3043                 instr->alu.src.offset = fsrc->offset / 8;
3044                 return 0;
3045         }
3046
3047         /* SHR_MI, SHR_HI. */
3048         src_val = strtoull(src, &src, 0);
3049         CHECK(!src[0], EINVAL);
3050
3051         instr->type = INSTR_ALU_SHR_MI;
3052         if (dst[0] == 'h')
3053                 instr->type = INSTR_ALU_SHR_HI;
3054
3055         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3056         instr->alu.dst.n_bits = fdst->n_bits;
3057         instr->alu.dst.offset = fdst->offset / 8;
3058         instr->alu.src_val = src_val;
3059         return 0;
3060 }
3061
3062 static int
3063 instr_alu_and_translate(struct rte_swx_pipeline *p,
3064                         struct action *action,
3065                         char **tokens,
3066                         int n_tokens,
3067                         struct instruction *instr,
3068                         struct instruction_data *data __rte_unused)
3069 {
3070         char *dst = tokens[1], *src = tokens[2];
3071         struct field *fdst, *fsrc;
3072         uint64_t src_val;
3073         uint32_t dst_struct_id = 0, src_struct_id = 0;
3074
3075         CHECK(n_tokens == 3, EINVAL);
3076
3077         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3078         CHECK(fdst, EINVAL);
3079         CHECK(!fdst->var_size, EINVAL);
3080
3081         /* AND, AND_MH, AND_HM, AND_HH. */
3082         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3083         if (fsrc) {
3084                 CHECK(!fsrc->var_size, EINVAL);
3085
3086                 instr->type = INSTR_ALU_AND;
3087                 if (dst[0] != 'h' && src[0] == 'h')
3088                         instr->type = INSTR_ALU_AND_MH;
3089                 if (dst[0] == 'h' && src[0] != 'h')
3090                         instr->type = INSTR_ALU_AND_HM;
3091                 if (dst[0] == 'h' && src[0] == 'h')
3092                         instr->type = INSTR_ALU_AND_HH;
3093
3094                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3095                 instr->alu.dst.n_bits = fdst->n_bits;
3096                 instr->alu.dst.offset = fdst->offset / 8;
3097                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3098                 instr->alu.src.n_bits = fsrc->n_bits;
3099                 instr->alu.src.offset = fsrc->offset / 8;
3100                 return 0;
3101         }
3102
3103         /* AND_I. */
3104         src_val = strtoull(src, &src, 0);
3105         CHECK(!src[0], EINVAL);
3106
3107         if (dst[0] == 'h')
3108                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3109
3110         instr->type = INSTR_ALU_AND_I;
3111         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3112         instr->alu.dst.n_bits = fdst->n_bits;
3113         instr->alu.dst.offset = fdst->offset / 8;
3114         instr->alu.src_val = src_val;
3115         return 0;
3116 }
3117
3118 static int
3119 instr_alu_or_translate(struct rte_swx_pipeline *p,
3120                        struct action *action,
3121                        char **tokens,
3122                        int n_tokens,
3123                        struct instruction *instr,
3124                        struct instruction_data *data __rte_unused)
3125 {
3126         char *dst = tokens[1], *src = tokens[2];
3127         struct field *fdst, *fsrc;
3128         uint64_t src_val;
3129         uint32_t dst_struct_id = 0, src_struct_id = 0;
3130
3131         CHECK(n_tokens == 3, EINVAL);
3132
3133         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3134         CHECK(fdst, EINVAL);
3135         CHECK(!fdst->var_size, EINVAL);
3136
3137         /* OR, OR_MH, OR_HM, OR_HH. */
3138         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3139         if (fsrc) {
3140                 CHECK(!fsrc->var_size, EINVAL);
3141
3142                 instr->type = INSTR_ALU_OR;
3143                 if (dst[0] != 'h' && src[0] == 'h')
3144                         instr->type = INSTR_ALU_OR_MH;
3145                 if (dst[0] == 'h' && src[0] != 'h')
3146                         instr->type = INSTR_ALU_OR_HM;
3147                 if (dst[0] == 'h' && src[0] == 'h')
3148                         instr->type = INSTR_ALU_OR_HH;
3149
3150                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3151                 instr->alu.dst.n_bits = fdst->n_bits;
3152                 instr->alu.dst.offset = fdst->offset / 8;
3153                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3154                 instr->alu.src.n_bits = fsrc->n_bits;
3155                 instr->alu.src.offset = fsrc->offset / 8;
3156                 return 0;
3157         }
3158
3159         /* OR_I. */
3160         src_val = strtoull(src, &src, 0);
3161         CHECK(!src[0], EINVAL);
3162
3163         if (dst[0] == 'h')
3164                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3165
3166         instr->type = INSTR_ALU_OR_I;
3167         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3168         instr->alu.dst.n_bits = fdst->n_bits;
3169         instr->alu.dst.offset = fdst->offset / 8;
3170         instr->alu.src_val = src_val;
3171         return 0;
3172 }
3173
3174 static int
3175 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3176                         struct action *action,
3177                         char **tokens,
3178                         int n_tokens,
3179                         struct instruction *instr,
3180                         struct instruction_data *data __rte_unused)
3181 {
3182         char *dst = tokens[1], *src = tokens[2];
3183         struct field *fdst, *fsrc;
3184         uint64_t src_val;
3185         uint32_t dst_struct_id = 0, src_struct_id = 0;
3186
3187         CHECK(n_tokens == 3, EINVAL);
3188
3189         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3190         CHECK(fdst, EINVAL);
3191         CHECK(!fdst->var_size, EINVAL);
3192
3193         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3194         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3195         if (fsrc) {
3196                 CHECK(!fsrc->var_size, EINVAL);
3197
3198                 instr->type = INSTR_ALU_XOR;
3199                 if (dst[0] != 'h' && src[0] == 'h')
3200                         instr->type = INSTR_ALU_XOR_MH;
3201                 if (dst[0] == 'h' && src[0] != 'h')
3202                         instr->type = INSTR_ALU_XOR_HM;
3203                 if (dst[0] == 'h' && src[0] == 'h')
3204                         instr->type = INSTR_ALU_XOR_HH;
3205
3206                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3207                 instr->alu.dst.n_bits = fdst->n_bits;
3208                 instr->alu.dst.offset = fdst->offset / 8;
3209                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3210                 instr->alu.src.n_bits = fsrc->n_bits;
3211                 instr->alu.src.offset = fsrc->offset / 8;
3212                 return 0;
3213         }
3214
3215         /* XOR_I. */
3216         src_val = strtoull(src, &src, 0);
3217         CHECK(!src[0], EINVAL);
3218
3219         if (dst[0] == 'h')
3220                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3221
3222         instr->type = INSTR_ALU_XOR_I;
3223         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3224         instr->alu.dst.n_bits = fdst->n_bits;
3225         instr->alu.dst.offset = fdst->offset / 8;
3226         instr->alu.src_val = src_val;
3227         return 0;
3228 }
3229
3230 static inline void
3231 instr_alu_add_exec(struct rte_swx_pipeline *p)
3232 {
3233         struct thread *t = &p->threads[p->thread_id];
3234         struct instruction *ip = t->ip;
3235
3236         /* Structs */
3237         __instr_alu_add_exec(p, t, ip);
3238
3239         /* Thread. */
3240         thread_ip_inc(p);
3241 }
3242
3243 static inline void
3244 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3245 {
3246         struct thread *t = &p->threads[p->thread_id];
3247         struct instruction *ip = t->ip;
3248
3249         /* Structs. */
3250         __instr_alu_add_mh_exec(p, t, ip);
3251
3252         /* Thread. */
3253         thread_ip_inc(p);
3254 }
3255
3256 static inline void
3257 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3258 {
3259         struct thread *t = &p->threads[p->thread_id];
3260         struct instruction *ip = t->ip;
3261
3262         /* Structs. */
3263         __instr_alu_add_hm_exec(p, t, ip);
3264
3265         /* Thread. */
3266         thread_ip_inc(p);
3267 }
3268
3269 static inline void
3270 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3271 {
3272         struct thread *t = &p->threads[p->thread_id];
3273         struct instruction *ip = t->ip;
3274
3275         /* Structs. */
3276         __instr_alu_add_hh_exec(p, t, ip);
3277
3278         /* Thread. */
3279         thread_ip_inc(p);
3280 }
3281
3282 static inline void
3283 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3284 {
3285         struct thread *t = &p->threads[p->thread_id];
3286         struct instruction *ip = t->ip;
3287
3288         /* Structs. */
3289         __instr_alu_add_mi_exec(p, t, ip);
3290
3291         /* Thread. */
3292         thread_ip_inc(p);
3293 }
3294
3295 static inline void
3296 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3297 {
3298         struct thread *t = &p->threads[p->thread_id];
3299         struct instruction *ip = t->ip;
3300
3301         /* Structs. */
3302         __instr_alu_add_hi_exec(p, t, ip);
3303
3304         /* Thread. */
3305         thread_ip_inc(p);
3306 }
3307
3308 static inline void
3309 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3310 {
3311         struct thread *t = &p->threads[p->thread_id];
3312         struct instruction *ip = t->ip;
3313
3314         /* Structs. */
3315         __instr_alu_sub_exec(p, t, ip);
3316
3317         /* Thread. */
3318         thread_ip_inc(p);
3319 }
3320
3321 static inline void
3322 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3323 {
3324         struct thread *t = &p->threads[p->thread_id];
3325         struct instruction *ip = t->ip;
3326
3327         /* Structs. */
3328         __instr_alu_sub_mh_exec(p, t, ip);
3329
3330         /* Thread. */
3331         thread_ip_inc(p);
3332 }
3333
3334 static inline void
3335 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3336 {
3337         struct thread *t = &p->threads[p->thread_id];
3338         struct instruction *ip = t->ip;
3339
3340         /* Structs. */
3341         __instr_alu_sub_hm_exec(p, t, ip);
3342
3343         /* Thread. */
3344         thread_ip_inc(p);
3345 }
3346
3347 static inline void
3348 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3349 {
3350         struct thread *t = &p->threads[p->thread_id];
3351         struct instruction *ip = t->ip;
3352
3353         /* Structs. */
3354         __instr_alu_sub_hh_exec(p, t, ip);
3355
3356         /* Thread. */
3357         thread_ip_inc(p);
3358 }
3359
3360 static inline void
3361 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3362 {
3363         struct thread *t = &p->threads[p->thread_id];
3364         struct instruction *ip = t->ip;
3365
3366         /* Structs. */
3367         __instr_alu_sub_mi_exec(p, t, ip);
3368
3369         /* Thread. */
3370         thread_ip_inc(p);
3371 }
3372
3373 static inline void
3374 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3375 {
3376         struct thread *t = &p->threads[p->thread_id];
3377         struct instruction *ip = t->ip;
3378
3379         /* Structs. */
3380         __instr_alu_sub_hi_exec(p, t, ip);
3381
3382         /* Thread. */
3383         thread_ip_inc(p);
3384 }
3385
3386 static inline void
3387 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3388 {
3389         struct thread *t = &p->threads[p->thread_id];
3390         struct instruction *ip = t->ip;
3391
3392         /* Structs. */
3393         __instr_alu_shl_exec(p, t, ip);
3394
3395         /* Thread. */
3396         thread_ip_inc(p);
3397 }
3398
3399 static inline void
3400 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3401 {
3402         struct thread *t = &p->threads[p->thread_id];
3403         struct instruction *ip = t->ip;
3404
3405         /* Structs. */
3406         __instr_alu_shl_mh_exec(p, t, ip);
3407
3408         /* Thread. */
3409         thread_ip_inc(p);
3410 }
3411
3412 static inline void
3413 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3414 {
3415         struct thread *t = &p->threads[p->thread_id];
3416         struct instruction *ip = t->ip;
3417
3418         /* Structs. */
3419         __instr_alu_shl_hm_exec(p, t, ip);
3420
3421         /* Thread. */
3422         thread_ip_inc(p);
3423 }
3424
3425 static inline void
3426 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3427 {
3428         struct thread *t = &p->threads[p->thread_id];
3429         struct instruction *ip = t->ip;
3430
3431         /* Structs. */
3432         __instr_alu_shl_hh_exec(p, t, ip);
3433
3434         /* Thread. */
3435         thread_ip_inc(p);
3436 }
3437
3438 static inline void
3439 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3440 {
3441         struct thread *t = &p->threads[p->thread_id];
3442         struct instruction *ip = t->ip;
3443
3444         /* Structs. */
3445         __instr_alu_shl_mi_exec(p, t, ip);
3446
3447         /* Thread. */
3448         thread_ip_inc(p);
3449 }
3450
3451 static inline void
3452 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3453 {
3454         struct thread *t = &p->threads[p->thread_id];
3455         struct instruction *ip = t->ip;
3456
3457         /* Structs. */
3458         __instr_alu_shl_hi_exec(p, t, ip);
3459
3460         /* Thread. */
3461         thread_ip_inc(p);
3462 }
3463
3464 static inline void
3465 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3466 {
3467         struct thread *t = &p->threads[p->thread_id];
3468         struct instruction *ip = t->ip;
3469
3470         /* Structs. */
3471         __instr_alu_shr_exec(p, t, ip);
3472
3473         /* Thread. */
3474         thread_ip_inc(p);
3475 }
3476
3477 static inline void
3478 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3479 {
3480         struct thread *t = &p->threads[p->thread_id];
3481         struct instruction *ip = t->ip;
3482
3483         /* Structs. */
3484         __instr_alu_shr_mh_exec(p, t, ip);
3485
3486         /* Thread. */
3487         thread_ip_inc(p);
3488 }
3489
3490 static inline void
3491 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3492 {
3493         struct thread *t = &p->threads[p->thread_id];
3494         struct instruction *ip = t->ip;
3495
3496         /* Structs. */
3497         __instr_alu_shr_hm_exec(p, t, ip);
3498
3499         /* Thread. */
3500         thread_ip_inc(p);
3501 }
3502
3503 static inline void
3504 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3505 {
3506         struct thread *t = &p->threads[p->thread_id];
3507         struct instruction *ip = t->ip;
3508
3509         /* Structs. */
3510         __instr_alu_shr_hh_exec(p, t, ip);
3511
3512         /* Thread. */
3513         thread_ip_inc(p);
3514 }
3515
3516 static inline void
3517 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3518 {
3519         struct thread *t = &p->threads[p->thread_id];
3520         struct instruction *ip = t->ip;
3521
3522         /* Structs. */
3523         __instr_alu_shr_mi_exec(p, t, ip);
3524
3525         /* Thread. */
3526         thread_ip_inc(p);
3527 }
3528
3529 static inline void
3530 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3531 {
3532         struct thread *t = &p->threads[p->thread_id];
3533         struct instruction *ip = t->ip;
3534
3535         /* Structs. */
3536         __instr_alu_shr_hi_exec(p, t, ip);
3537
3538         /* Thread. */
3539         thread_ip_inc(p);
3540 }
3541
3542 static inline void
3543 instr_alu_and_exec(struct rte_swx_pipeline *p)
3544 {
3545         struct thread *t = &p->threads[p->thread_id];
3546         struct instruction *ip = t->ip;
3547
3548         /* Structs. */
3549         __instr_alu_and_exec(p, t, ip);
3550
3551         /* Thread. */
3552         thread_ip_inc(p);
3553 }
3554
3555 static inline void
3556 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3557 {
3558         struct thread *t = &p->threads[p->thread_id];
3559         struct instruction *ip = t->ip;
3560
3561         /* Structs. */
3562         __instr_alu_and_mh_exec(p, t, ip);
3563
3564         /* Thread. */
3565         thread_ip_inc(p);
3566 }
3567
3568 static inline void
3569 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3570 {
3571         struct thread *t = &p->threads[p->thread_id];
3572         struct instruction *ip = t->ip;
3573
3574         /* Structs. */
3575         __instr_alu_and_hm_exec(p, t, ip);
3576
3577         /* Thread. */
3578         thread_ip_inc(p);
3579 }
3580
3581 static inline void
3582 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3583 {
3584         struct thread *t = &p->threads[p->thread_id];
3585         struct instruction *ip = t->ip;
3586
3587         /* Structs. */
3588         __instr_alu_and_hh_exec(p, t, ip);
3589
3590         /* Thread. */
3591         thread_ip_inc(p);
3592 }
3593
3594 static inline void
3595 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3596 {
3597         struct thread *t = &p->threads[p->thread_id];
3598         struct instruction *ip = t->ip;
3599
3600         /* Structs. */
3601         __instr_alu_and_i_exec(p, t, ip);
3602
3603         /* Thread. */
3604         thread_ip_inc(p);
3605 }
3606
3607 static inline void
3608 instr_alu_or_exec(struct rte_swx_pipeline *p)
3609 {
3610         struct thread *t = &p->threads[p->thread_id];
3611         struct instruction *ip = t->ip;
3612
3613         /* Structs. */
3614         __instr_alu_or_exec(p, t, ip);
3615
3616         /* Thread. */
3617         thread_ip_inc(p);
3618 }
3619
3620 static inline void
3621 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3622 {
3623         struct thread *t = &p->threads[p->thread_id];
3624         struct instruction *ip = t->ip;
3625
3626         /* Structs. */
3627         __instr_alu_or_mh_exec(p, t, ip);
3628
3629         /* Thread. */
3630         thread_ip_inc(p);
3631 }
3632
3633 static inline void
3634 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3635 {
3636         struct thread *t = &p->threads[p->thread_id];
3637         struct instruction *ip = t->ip;
3638
3639         /* Structs. */
3640         __instr_alu_or_hm_exec(p, t, ip);
3641
3642         /* Thread. */
3643         thread_ip_inc(p);
3644 }
3645
3646 static inline void
3647 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3648 {
3649         struct thread *t = &p->threads[p->thread_id];
3650         struct instruction *ip = t->ip;
3651
3652         /* Structs. */
3653         __instr_alu_or_hh_exec(p, t, ip);
3654
3655         /* Thread. */
3656         thread_ip_inc(p);
3657 }
3658
3659 static inline void
3660 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3661 {
3662         struct thread *t = &p->threads[p->thread_id];
3663         struct instruction *ip = t->ip;
3664
3665         /* Structs. */
3666         __instr_alu_or_i_exec(p, t, ip);
3667
3668         /* Thread. */
3669         thread_ip_inc(p);
3670 }
3671
3672 static inline void
3673 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3674 {
3675         struct thread *t = &p->threads[p->thread_id];
3676         struct instruction *ip = t->ip;
3677
3678         /* Structs. */
3679         __instr_alu_xor_exec(p, t, ip);
3680
3681         /* Thread. */
3682         thread_ip_inc(p);
3683 }
3684
3685 static inline void
3686 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3687 {
3688         struct thread *t = &p->threads[p->thread_id];
3689         struct instruction *ip = t->ip;
3690
3691         /* Structs. */
3692         __instr_alu_xor_mh_exec(p, t, ip);
3693
3694         /* Thread. */
3695         thread_ip_inc(p);
3696 }
3697
3698 static inline void
3699 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3700 {
3701         struct thread *t = &p->threads[p->thread_id];
3702         struct instruction *ip = t->ip;
3703
3704         /* Structs. */
3705         __instr_alu_xor_hm_exec(p, t, ip);
3706
3707         /* Thread. */
3708         thread_ip_inc(p);
3709 }
3710
3711 static inline void
3712 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3713 {
3714         struct thread *t = &p->threads[p->thread_id];
3715         struct instruction *ip = t->ip;
3716
3717         /* Structs. */
3718         __instr_alu_xor_hh_exec(p, t, ip);
3719
3720         /* Thread. */
3721         thread_ip_inc(p);
3722 }
3723
3724 static inline void
3725 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3726 {
3727         struct thread *t = &p->threads[p->thread_id];
3728         struct instruction *ip = t->ip;
3729
3730         /* Structs. */
3731         __instr_alu_xor_i_exec(p, t, ip);
3732
3733         /* Thread. */
3734         thread_ip_inc(p);
3735 }
3736
3737 static inline void
3738 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3739 {
3740         struct thread *t = &p->threads[p->thread_id];
3741         struct instruction *ip = t->ip;
3742
3743         /* Structs. */
3744         __instr_alu_ckadd_field_exec(p, t, ip);
3745
3746         /* Thread. */
3747         thread_ip_inc(p);
3748 }
3749
3750 static inline void
3751 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3752 {
3753         struct thread *t = &p->threads[p->thread_id];
3754         struct instruction *ip = t->ip;
3755
3756         /* Structs. */
3757         __instr_alu_cksub_field_exec(p, t, ip);
3758
3759         /* Thread. */
3760         thread_ip_inc(p);
3761 }
3762
3763 static inline void
3764 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3765 {
3766         struct thread *t = &p->threads[p->thread_id];
3767         struct instruction *ip = t->ip;
3768
3769         /* Structs. */
3770         __instr_alu_ckadd_struct20_exec(p, t, ip);
3771
3772         /* Thread. */
3773         thread_ip_inc(p);
3774 }
3775
3776 static inline void
3777 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3778 {
3779         struct thread *t = &p->threads[p->thread_id];
3780         struct instruction *ip = t->ip;
3781
3782         /* Structs. */
3783         __instr_alu_ckadd_struct_exec(p, t, ip);
3784
3785         /* Thread. */
3786         thread_ip_inc(p);
3787 }
3788
3789 /*
3790  * Register array.
3791  */
3792 static struct regarray *
3793 regarray_find(struct rte_swx_pipeline *p, const char *name);
3794
3795 static int
3796 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3797                       struct action *action,
3798                       char **tokens,
3799                       int n_tokens,
3800                       struct instruction *instr,
3801                       struct instruction_data *data __rte_unused)
3802 {
3803         char *regarray = tokens[1], *idx = tokens[2];
3804         struct regarray *r;
3805         struct field *fidx;
3806         uint32_t idx_struct_id, idx_val;
3807
3808         CHECK(n_tokens == 3, EINVAL);
3809
3810         r = regarray_find(p, regarray);
3811         CHECK(r, EINVAL);
3812
3813         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3814         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3815         if (fidx) {
3816                 CHECK(!fidx->var_size, EINVAL);
3817
3818                 instr->type = INSTR_REGPREFETCH_RM;
3819                 if (idx[0] == 'h')
3820                         instr->type = INSTR_REGPREFETCH_RH;
3821
3822                 instr->regarray.regarray_id = r->id;
3823                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3824                 instr->regarray.idx.n_bits = fidx->n_bits;
3825                 instr->regarray.idx.offset = fidx->offset / 8;
3826                 instr->regarray.dstsrc_val = 0; /* Unused. */
3827                 return 0;
3828         }
3829
3830         /* REGPREFETCH_RI. */
3831         idx_val = strtoul(idx, &idx, 0);
3832         CHECK(!idx[0], EINVAL);
3833
3834         instr->type = INSTR_REGPREFETCH_RI;
3835         instr->regarray.regarray_id = r->id;
3836         instr->regarray.idx_val = idx_val;
3837         instr->regarray.dstsrc_val = 0; /* Unused. */
3838         return 0;
3839 }
3840
3841 static int
3842 instr_regrd_translate(struct rte_swx_pipeline *p,
3843                       struct action *action,
3844                       char **tokens,
3845                       int n_tokens,
3846                       struct instruction *instr,
3847                       struct instruction_data *data __rte_unused)
3848 {
3849         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3850         struct regarray *r;
3851         struct field *fdst, *fidx;
3852         uint32_t dst_struct_id, idx_struct_id, idx_val;
3853
3854         CHECK(n_tokens == 4, EINVAL);
3855
3856         r = regarray_find(p, regarray);
3857         CHECK(r, EINVAL);
3858
3859         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3860         CHECK(fdst, EINVAL);
3861         CHECK(!fdst->var_size, EINVAL);
3862
3863         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3864         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3865         if (fidx) {
3866                 CHECK(!fidx->var_size, EINVAL);
3867
3868                 instr->type = INSTR_REGRD_MRM;
3869                 if (dst[0] == 'h' && idx[0] != 'h')
3870                         instr->type = INSTR_REGRD_HRM;
3871                 if (dst[0] != 'h' && idx[0] == 'h')
3872                         instr->type = INSTR_REGRD_MRH;
3873                 if (dst[0] == 'h' && idx[0] == 'h')
3874                         instr->type = INSTR_REGRD_HRH;
3875
3876                 instr->regarray.regarray_id = r->id;
3877                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3878                 instr->regarray.idx.n_bits = fidx->n_bits;
3879                 instr->regarray.idx.offset = fidx->offset / 8;
3880                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3881                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3882                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3883                 return 0;
3884         }
3885
3886         /* REGRD_MRI, REGRD_HRI. */
3887         idx_val = strtoul(idx, &idx, 0);
3888         CHECK(!idx[0], EINVAL);
3889
3890         instr->type = INSTR_REGRD_MRI;
3891         if (dst[0] == 'h')
3892                 instr->type = INSTR_REGRD_HRI;
3893
3894         instr->regarray.regarray_id = r->id;
3895         instr->regarray.idx_val = idx_val;
3896         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3897         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3898         instr->regarray.dstsrc.offset = fdst->offset / 8;
3899         return 0;
3900 }
3901
3902 static int
3903 instr_regwr_translate(struct rte_swx_pipeline *p,
3904                       struct action *action,
3905                       char **tokens,
3906                       int n_tokens,
3907                       struct instruction *instr,
3908                       struct instruction_data *data __rte_unused)
3909 {
3910         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3911         struct regarray *r;
3912         struct field *fidx, *fsrc;
3913         uint64_t src_val;
3914         uint32_t idx_struct_id, idx_val, src_struct_id;
3915
3916         CHECK(n_tokens == 4, EINVAL);
3917
3918         r = regarray_find(p, regarray);
3919         CHECK(r, EINVAL);
3920
3921         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3922         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3923         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3924         if (fidx && fsrc) {
3925                 CHECK(!fidx->var_size, EINVAL);
3926                 CHECK(!fsrc->var_size, EINVAL);
3927
3928                 instr->type = INSTR_REGWR_RMM;
3929                 if (idx[0] == 'h' && src[0] != 'h')
3930                         instr->type = INSTR_REGWR_RHM;
3931                 if (idx[0] != 'h' && src[0] == 'h')
3932                         instr->type = INSTR_REGWR_RMH;
3933                 if (idx[0] == 'h' && src[0] == 'h')
3934                         instr->type = INSTR_REGWR_RHH;
3935
3936                 instr->regarray.regarray_id = r->id;
3937                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3938                 instr->regarray.idx.n_bits = fidx->n_bits;
3939                 instr->regarray.idx.offset = fidx->offset / 8;
3940                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3941                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3942                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3943                 return 0;
3944         }
3945
3946         /* REGWR_RHI, REGWR_RMI. */
3947         if (fidx && !fsrc) {
3948                 CHECK(!fidx->var_size, EINVAL);
3949
3950                 src_val = strtoull(src, &src, 0);
3951                 CHECK(!src[0], EINVAL);
3952
3953                 instr->type = INSTR_REGWR_RMI;
3954                 if (idx[0] == 'h')
3955                         instr->type = INSTR_REGWR_RHI;
3956
3957                 instr->regarray.regarray_id = r->id;
3958                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3959                 instr->regarray.idx.n_bits = fidx->n_bits;
3960                 instr->regarray.idx.offset = fidx->offset / 8;
3961                 instr->regarray.dstsrc_val = src_val;
3962                 return 0;
3963         }
3964
3965         /* REGWR_RIH, REGWR_RIM. */
3966         if (!fidx && fsrc) {
3967                 idx_val = strtoul(idx, &idx, 0);
3968                 CHECK(!idx[0], EINVAL);
3969
3970                 CHECK(!fsrc->var_size, EINVAL);
3971
3972                 instr->type = INSTR_REGWR_RIM;
3973                 if (src[0] == 'h')
3974                         instr->type = INSTR_REGWR_RIH;
3975
3976                 instr->regarray.regarray_id = r->id;
3977                 instr->regarray.idx_val = idx_val;
3978                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3979                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3980                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3981                 return 0;
3982         }
3983
3984         /* REGWR_RII. */
3985         src_val = strtoull(src, &src, 0);
3986         CHECK(!src[0], EINVAL);
3987
3988         idx_val = strtoul(idx, &idx, 0);
3989         CHECK(!idx[0], EINVAL);
3990
3991         instr->type = INSTR_REGWR_RII;
3992         instr->regarray.idx_val = idx_val;
3993         instr->regarray.dstsrc_val = src_val;
3994
3995         return 0;
3996 }
3997
3998 static int
3999 instr_regadd_translate(struct rte_swx_pipeline *p,
4000                        struct action *action,
4001                        char **tokens,
4002                        int n_tokens,
4003                        struct instruction *instr,
4004                        struct instruction_data *data __rte_unused)
4005 {
4006         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4007         struct regarray *r;
4008         struct field *fidx, *fsrc;
4009         uint64_t src_val;
4010         uint32_t idx_struct_id, idx_val, src_struct_id;
4011
4012         CHECK(n_tokens == 4, EINVAL);
4013
4014         r = regarray_find(p, regarray);
4015         CHECK(r, EINVAL);
4016
4017         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4018         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4019         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4020         if (fidx && fsrc) {
4021                 CHECK(!fidx->var_size, EINVAL);
4022                 CHECK(!fsrc->var_size, EINVAL);
4023
4024                 instr->type = INSTR_REGADD_RMM;
4025                 if (idx[0] == 'h' && src[0] != 'h')
4026                         instr->type = INSTR_REGADD_RHM;
4027                 if (idx[0] != 'h' && src[0] == 'h')
4028                         instr->type = INSTR_REGADD_RMH;
4029                 if (idx[0] == 'h' && src[0] == 'h')
4030                         instr->type = INSTR_REGADD_RHH;
4031
4032                 instr->regarray.regarray_id = r->id;
4033                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4034                 instr->regarray.idx.n_bits = fidx->n_bits;
4035                 instr->regarray.idx.offset = fidx->offset / 8;
4036                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4037                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4038                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4039                 return 0;
4040         }
4041
4042         /* REGADD_RHI, REGADD_RMI. */
4043         if (fidx && !fsrc) {
4044                 CHECK(!fidx->var_size, EINVAL);
4045
4046                 src_val = strtoull(src, &src, 0);
4047                 CHECK(!src[0], EINVAL);
4048
4049                 instr->type = INSTR_REGADD_RMI;
4050                 if (idx[0] == 'h')
4051                         instr->type = INSTR_REGADD_RHI;
4052
4053                 instr->regarray.regarray_id = r->id;
4054                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4055                 instr->regarray.idx.n_bits = fidx->n_bits;
4056                 instr->regarray.idx.offset = fidx->offset / 8;
4057                 instr->regarray.dstsrc_val = src_val;
4058                 return 0;
4059         }
4060
4061         /* REGADD_RIH, REGADD_RIM. */
4062         if (!fidx && fsrc) {
4063                 idx_val = strtoul(idx, &idx, 0);
4064                 CHECK(!idx[0], EINVAL);
4065
4066                 CHECK(!fsrc->var_size, EINVAL);
4067
4068                 instr->type = INSTR_REGADD_RIM;
4069                 if (src[0] == 'h')
4070                         instr->type = INSTR_REGADD_RIH;
4071
4072                 instr->regarray.regarray_id = r->id;
4073                 instr->regarray.idx_val = idx_val;
4074                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4075                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4076                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4077                 return 0;
4078         }
4079
4080         /* REGADD_RII. */
4081         src_val = strtoull(src, &src, 0);
4082         CHECK(!src[0], EINVAL);
4083
4084         idx_val = strtoul(idx, &idx, 0);
4085         CHECK(!idx[0], EINVAL);
4086
4087         instr->type = INSTR_REGADD_RII;
4088         instr->regarray.idx_val = idx_val;
4089         instr->regarray.dstsrc_val = src_val;
4090         return 0;
4091 }
4092
4093 static inline void
4094 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4095 {
4096         struct thread *t = &p->threads[p->thread_id];
4097         struct instruction *ip = t->ip;
4098
4099         /* Structs. */
4100         __instr_regprefetch_rh_exec(p, t, ip);
4101
4102         /* Thread. */
4103         thread_ip_inc(p);
4104 }
4105
4106 static inline void
4107 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4108 {
4109         struct thread *t = &p->threads[p->thread_id];
4110         struct instruction *ip = t->ip;
4111
4112         /* Structs. */
4113         __instr_regprefetch_rm_exec(p, t, ip);
4114
4115         /* Thread. */
4116         thread_ip_inc(p);
4117 }
4118
4119 static inline void
4120 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4121 {
4122         struct thread *t = &p->threads[p->thread_id];
4123         struct instruction *ip = t->ip;
4124
4125         /* Structs. */
4126         __instr_regprefetch_ri_exec(p, t, ip);
4127
4128         /* Thread. */
4129         thread_ip_inc(p);
4130 }
4131
4132 static inline void
4133 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4134 {
4135         struct thread *t = &p->threads[p->thread_id];
4136         struct instruction *ip = t->ip;
4137
4138         /* Structs. */
4139         __instr_regrd_hrh_exec(p, t, ip);
4140
4141         /* Thread. */
4142         thread_ip_inc(p);
4143 }
4144
4145 static inline void
4146 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4147 {
4148         struct thread *t = &p->threads[p->thread_id];
4149         struct instruction *ip = t->ip;
4150
4151         /* Structs. */
4152         __instr_regrd_hrm_exec(p, t, ip);
4153
4154         /* Thread. */
4155         thread_ip_inc(p);
4156 }
4157
4158 static inline void
4159 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4160 {
4161         struct thread *t = &p->threads[p->thread_id];
4162         struct instruction *ip = t->ip;
4163
4164         /* Structs. */
4165         __instr_regrd_mrh_exec(p, t, ip);
4166
4167         /* Thread. */
4168         thread_ip_inc(p);
4169 }
4170
4171 static inline void
4172 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4173 {
4174         struct thread *t = &p->threads[p->thread_id];
4175         struct instruction *ip = t->ip;
4176
4177         /* Structs. */
4178         __instr_regrd_mrm_exec(p, t, ip);
4179
4180         /* Thread. */
4181         thread_ip_inc(p);
4182 }
4183
4184 static inline void
4185 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4186 {
4187         struct thread *t = &p->threads[p->thread_id];
4188         struct instruction *ip = t->ip;
4189
4190         /* Structs. */
4191         __instr_regrd_hri_exec(p, t, ip);
4192
4193         /* Thread. */
4194         thread_ip_inc(p);
4195 }
4196
4197 static inline void
4198 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4199 {
4200         struct thread *t = &p->threads[p->thread_id];
4201         struct instruction *ip = t->ip;
4202
4203         /* Structs. */
4204         __instr_regrd_mri_exec(p, t, ip);
4205
4206         /* Thread. */
4207         thread_ip_inc(p);
4208 }
4209
4210 static inline void
4211 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4212 {
4213         struct thread *t = &p->threads[p->thread_id];
4214         struct instruction *ip = t->ip;
4215
4216         /* Structs. */
4217         __instr_regwr_rhh_exec(p, t, ip);
4218
4219         /* Thread. */
4220         thread_ip_inc(p);
4221 }
4222
4223 static inline void
4224 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4225 {
4226         struct thread *t = &p->threads[p->thread_id];
4227         struct instruction *ip = t->ip;
4228
4229         /* Structs. */
4230         __instr_regwr_rhm_exec(p, t, ip);
4231
4232         /* Thread. */
4233         thread_ip_inc(p);
4234 }
4235
4236 static inline void
4237 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4238 {
4239         struct thread *t = &p->threads[p->thread_id];
4240         struct instruction *ip = t->ip;
4241
4242         /* Structs. */
4243         __instr_regwr_rmh_exec(p, t, ip);
4244
4245         /* Thread. */
4246         thread_ip_inc(p);
4247 }
4248
4249 static inline void
4250 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4251 {
4252         struct thread *t = &p->threads[p->thread_id];
4253         struct instruction *ip = t->ip;
4254
4255         /* Structs. */
4256         __instr_regwr_rmm_exec(p, t, ip);
4257
4258         /* Thread. */
4259         thread_ip_inc(p);
4260 }
4261
4262 static inline void
4263 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4264 {
4265         struct thread *t = &p->threads[p->thread_id];
4266         struct instruction *ip = t->ip;
4267
4268         /* Structs. */
4269         __instr_regwr_rhi_exec(p, t, ip);
4270
4271         /* Thread. */
4272         thread_ip_inc(p);
4273 }
4274
4275 static inline void
4276 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4277 {
4278         struct thread *t = &p->threads[p->thread_id];
4279         struct instruction *ip = t->ip;
4280
4281         /* Structs. */
4282         __instr_regwr_rmi_exec(p, t, ip);
4283
4284         /* Thread. */
4285         thread_ip_inc(p);
4286 }
4287
4288 static inline void
4289 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4290 {
4291         struct thread *t = &p->threads[p->thread_id];
4292         struct instruction *ip = t->ip;
4293
4294         /* Structs. */
4295         __instr_regwr_rih_exec(p, t, ip);
4296
4297         /* Thread. */
4298         thread_ip_inc(p);
4299 }
4300
4301 static inline void
4302 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4303 {
4304         struct thread *t = &p->threads[p->thread_id];
4305         struct instruction *ip = t->ip;
4306
4307         /* Structs. */
4308         __instr_regwr_rim_exec(p, t, ip);
4309
4310         /* Thread. */
4311         thread_ip_inc(p);
4312 }
4313
4314 static inline void
4315 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4316 {
4317         struct thread *t = &p->threads[p->thread_id];
4318         struct instruction *ip = t->ip;
4319
4320         /* Structs. */
4321         __instr_regwr_rii_exec(p, t, ip);
4322
4323         /* Thread. */
4324         thread_ip_inc(p);
4325 }
4326
4327 static inline void
4328 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4329 {
4330         struct thread *t = &p->threads[p->thread_id];
4331         struct instruction *ip = t->ip;
4332
4333         /* Structs. */
4334         __instr_regadd_rhh_exec(p, t, ip);
4335
4336         /* Thread. */
4337         thread_ip_inc(p);
4338 }
4339
4340 static inline void
4341 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4342 {
4343         struct thread *t = &p->threads[p->thread_id];
4344         struct instruction *ip = t->ip;
4345
4346         /* Structs. */
4347         __instr_regadd_rhm_exec(p, t, ip);
4348
4349         /* Thread. */
4350         thread_ip_inc(p);
4351 }
4352
4353 static inline void
4354 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4355 {
4356         struct thread *t = &p->threads[p->thread_id];
4357         struct instruction *ip = t->ip;
4358
4359         /* Structs. */
4360         __instr_regadd_rmh_exec(p, t, ip);
4361
4362         /* Thread. */
4363         thread_ip_inc(p);
4364 }
4365
4366 static inline void
4367 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4368 {
4369         struct thread *t = &p->threads[p->thread_id];
4370         struct instruction *ip = t->ip;
4371
4372         /* Structs. */
4373         __instr_regadd_rmm_exec(p, t, ip);
4374
4375         /* Thread. */
4376         thread_ip_inc(p);
4377 }
4378
4379 static inline void
4380 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4381 {
4382         struct thread *t = &p->threads[p->thread_id];
4383         struct instruction *ip = t->ip;
4384
4385         /* Structs. */
4386         __instr_regadd_rhi_exec(p, t, ip);
4387
4388         /* Thread. */
4389         thread_ip_inc(p);
4390 }
4391
4392 static inline void
4393 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4394 {
4395         struct thread *t = &p->threads[p->thread_id];
4396         struct instruction *ip = t->ip;
4397
4398         /* Structs. */
4399         __instr_regadd_rmi_exec(p, t, ip);
4400
4401         /* Thread. */
4402         thread_ip_inc(p);
4403 }
4404
4405 static inline void
4406 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4407 {
4408         struct thread *t = &p->threads[p->thread_id];
4409         struct instruction *ip = t->ip;
4410
4411         /* Structs. */
4412         __instr_regadd_rih_exec(p, t, ip);
4413
4414         /* Thread. */
4415         thread_ip_inc(p);
4416 }
4417
4418 static inline void
4419 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4420 {
4421         struct thread *t = &p->threads[p->thread_id];
4422         struct instruction *ip = t->ip;
4423
4424         /* Structs. */
4425         __instr_regadd_rim_exec(p, t, ip);
4426
4427         /* Thread. */
4428         thread_ip_inc(p);
4429 }
4430
4431 static inline void
4432 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4433 {
4434         struct thread *t = &p->threads[p->thread_id];
4435         struct instruction *ip = t->ip;
4436
4437         /* Structs. */
4438         __instr_regadd_rii_exec(p, t, ip);
4439
4440         /* Thread. */
4441         thread_ip_inc(p);
4442 }
4443
4444 /*
4445  * metarray.
4446  */
4447 static struct metarray *
4448 metarray_find(struct rte_swx_pipeline *p, const char *name);
4449
4450 static int
4451 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4452                             struct action *action,
4453                             char **tokens,
4454                             int n_tokens,
4455                             struct instruction *instr,
4456                             struct instruction_data *data __rte_unused)
4457 {
4458         char *metarray = tokens[1], *idx = tokens[2];
4459         struct metarray *m;
4460         struct field *fidx;
4461         uint32_t idx_struct_id, idx_val;
4462
4463         CHECK(n_tokens == 3, EINVAL);
4464
4465         m = metarray_find(p, metarray);
4466         CHECK(m, EINVAL);
4467
4468         /* METPREFETCH_H, METPREFETCH_M. */
4469         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4470         if (fidx) {
4471                 CHECK(!fidx->var_size, EINVAL);
4472
4473                 instr->type = INSTR_METPREFETCH_M;
4474                 if (idx[0] == 'h')
4475                         instr->type = INSTR_METPREFETCH_H;
4476
4477                 instr->meter.metarray_id = m->id;
4478                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4479                 instr->meter.idx.n_bits = fidx->n_bits;
4480                 instr->meter.idx.offset = fidx->offset / 8;
4481                 return 0;
4482         }
4483
4484         /* METPREFETCH_I. */
4485         idx_val = strtoul(idx, &idx, 0);
4486         CHECK(!idx[0], EINVAL);
4487
4488         instr->type = INSTR_METPREFETCH_I;
4489         instr->meter.metarray_id = m->id;
4490         instr->meter.idx_val = idx_val;
4491         return 0;
4492 }
4493
4494 static int
4495 instr_meter_translate(struct rte_swx_pipeline *p,
4496                       struct action *action,
4497                       char **tokens,
4498                       int n_tokens,
4499                       struct instruction *instr,
4500                       struct instruction_data *data __rte_unused)
4501 {
4502         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4503         char *color_in = tokens[4], *color_out = tokens[5];
4504         struct metarray *m;
4505         struct field *fidx, *flength, *fcin, *fcout;
4506         uint32_t idx_struct_id, length_struct_id;
4507         uint32_t color_in_struct_id, color_out_struct_id;
4508
4509         CHECK(n_tokens == 6, EINVAL);
4510
4511         m = metarray_find(p, metarray);
4512         CHECK(m, EINVAL);
4513
4514         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4515
4516         flength = struct_field_parse(p, action, length, &length_struct_id);
4517         CHECK(flength, EINVAL);
4518         CHECK(!flength->var_size, EINVAL);
4519
4520         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4521
4522         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4523         CHECK(fcout, EINVAL);
4524         CHECK(!fcout->var_size, EINVAL);
4525
4526         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4527         if (fidx && fcin) {
4528                 CHECK(!fidx->var_size, EINVAL);
4529                 CHECK(!fcin->var_size, EINVAL);
4530
4531                 instr->type = INSTR_METER_MMM;
4532                 if (idx[0] == 'h' && length[0] == 'h')
4533                         instr->type = INSTR_METER_HHM;
4534                 if (idx[0] == 'h' && length[0] != 'h')
4535                         instr->type = INSTR_METER_HMM;
4536                 if (idx[0] != 'h' && length[0] == 'h')
4537                         instr->type = INSTR_METER_MHM;
4538
4539                 instr->meter.metarray_id = m->id;
4540
4541                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4542                 instr->meter.idx.n_bits = fidx->n_bits;
4543                 instr->meter.idx.offset = fidx->offset / 8;
4544
4545                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4546                 instr->meter.length.n_bits = flength->n_bits;
4547                 instr->meter.length.offset = flength->offset / 8;
4548
4549                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4550                 instr->meter.color_in.n_bits = fcin->n_bits;
4551                 instr->meter.color_in.offset = fcin->offset / 8;
4552
4553                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4554                 instr->meter.color_out.n_bits = fcout->n_bits;
4555                 instr->meter.color_out.offset = fcout->offset / 8;
4556         }
4557
4558         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4559         if (fidx && !fcin) {
4560                 uint32_t color_in_val;
4561
4562                 CHECK(!fidx->var_size, EINVAL);
4563
4564                 color_in_val = strtoul(color_in, &color_in, 0);
4565                 CHECK(!color_in[0], EINVAL);
4566
4567                 instr->type = INSTR_METER_MMI;
4568                 if (idx[0] == 'h' && length[0] == 'h')
4569                         instr->type = INSTR_METER_HHI;
4570                 if (idx[0] == 'h' && length[0] != 'h')
4571                         instr->type = INSTR_METER_HMI;
4572                 if (idx[0] != 'h' && length[0] == 'h')
4573                         instr->type = INSTR_METER_MHI;
4574
4575                 instr->meter.metarray_id = m->id;
4576
4577                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4578                 instr->meter.idx.n_bits = fidx->n_bits;
4579                 instr->meter.idx.offset = fidx->offset / 8;
4580
4581                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4582                 instr->meter.length.n_bits = flength->n_bits;
4583                 instr->meter.length.offset = flength->offset / 8;
4584
4585                 instr->meter.color_in_val = color_in_val;
4586
4587                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4588                 instr->meter.color_out.n_bits = fcout->n_bits;
4589                 instr->meter.color_out.offset = fcout->offset / 8;
4590         }
4591
4592         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4593         if (!fidx && fcin) {
4594                 uint32_t idx_val;
4595
4596                 idx_val = strtoul(idx, &idx, 0);
4597                 CHECK(!idx[0], EINVAL);
4598
4599                 CHECK(!fcin->var_size, EINVAL);
4600
4601                 instr->type = INSTR_METER_IMM;
4602                 if (length[0] == 'h')
4603                         instr->type = INSTR_METER_IHM;
4604
4605                 instr->meter.metarray_id = m->id;
4606
4607                 instr->meter.idx_val = idx_val;
4608
4609                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4610                 instr->meter.length.n_bits = flength->n_bits;
4611                 instr->meter.length.offset = flength->offset / 8;
4612
4613                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4614                 instr->meter.color_in.n_bits = fcin->n_bits;
4615                 instr->meter.color_in.offset = fcin->offset / 8;
4616
4617                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4618                 instr->meter.color_out.n_bits = fcout->n_bits;
4619                 instr->meter.color_out.offset = fcout->offset / 8;
4620         }
4621
4622         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4623         if (!fidx && !fcin) {
4624                 uint32_t idx_val, color_in_val;
4625
4626                 idx_val = strtoul(idx, &idx, 0);
4627                 CHECK(!idx[0], EINVAL);
4628
4629                 color_in_val = strtoul(color_in, &color_in, 0);
4630                 CHECK(!color_in[0], EINVAL);
4631
4632                 instr->type = INSTR_METER_IMI;
4633                 if (length[0] == 'h')
4634                         instr->type = INSTR_METER_IHI;
4635
4636                 instr->meter.metarray_id = m->id;
4637
4638                 instr->meter.idx_val = idx_val;
4639
4640                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4641                 instr->meter.length.n_bits = flength->n_bits;
4642                 instr->meter.length.offset = flength->offset / 8;
4643
4644                 instr->meter.color_in_val = color_in_val;
4645
4646                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4647                 instr->meter.color_out.n_bits = fcout->n_bits;
4648                 instr->meter.color_out.offset = fcout->offset / 8;
4649         }
4650
4651         return 0;
4652 }
4653
4654 static inline void
4655 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4656 {
4657         struct thread *t = &p->threads[p->thread_id];
4658         struct instruction *ip = t->ip;
4659
4660         /* Structs. */
4661         __instr_metprefetch_h_exec(p, t, ip);
4662
4663         /* Thread. */
4664         thread_ip_inc(p);
4665 }
4666
4667 static inline void
4668 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4669 {
4670         struct thread *t = &p->threads[p->thread_id];
4671         struct instruction *ip = t->ip;
4672
4673         /* Structs. */
4674         __instr_metprefetch_m_exec(p, t, ip);
4675
4676         /* Thread. */
4677         thread_ip_inc(p);
4678 }
4679
4680 static inline void
4681 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4682 {
4683         struct thread *t = &p->threads[p->thread_id];
4684         struct instruction *ip = t->ip;
4685
4686         /* Structs. */
4687         __instr_metprefetch_i_exec(p, t, ip);
4688
4689         /* Thread. */
4690         thread_ip_inc(p);
4691 }
4692
4693 static inline void
4694 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4695 {
4696         struct thread *t = &p->threads[p->thread_id];
4697         struct instruction *ip = t->ip;
4698
4699         /* Structs. */
4700         __instr_meter_hhm_exec(p, t, ip);
4701
4702         /* Thread. */
4703         thread_ip_inc(p);
4704 }
4705
4706 static inline void
4707 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4708 {
4709         struct thread *t = &p->threads[p->thread_id];
4710         struct instruction *ip = t->ip;
4711
4712         /* Structs. */
4713         __instr_meter_hhi_exec(p, t, ip);
4714
4715         /* Thread. */
4716         thread_ip_inc(p);
4717 }
4718
4719 static inline void
4720 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4721 {
4722         struct thread *t = &p->threads[p->thread_id];
4723         struct instruction *ip = t->ip;
4724
4725         /* Structs. */
4726         __instr_meter_hmm_exec(p, t, ip);
4727
4728         /* Thread. */
4729         thread_ip_inc(p);
4730 }
4731
4732 static inline void
4733 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4734 {
4735         struct thread *t = &p->threads[p->thread_id];
4736         struct instruction *ip = t->ip;
4737
4738         /* Structs. */
4739         __instr_meter_hmi_exec(p, t, ip);
4740
4741         /* Thread. */
4742         thread_ip_inc(p);
4743 }
4744
4745 static inline void
4746 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4747 {
4748         struct thread *t = &p->threads[p->thread_id];
4749         struct instruction *ip = t->ip;
4750
4751         /* Structs. */
4752         __instr_meter_mhm_exec(p, t, ip);
4753
4754         /* Thread. */
4755         thread_ip_inc(p);
4756 }
4757
4758 static inline void
4759 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4760 {
4761         struct thread *t = &p->threads[p->thread_id];
4762         struct instruction *ip = t->ip;
4763
4764         /* Structs. */
4765         __instr_meter_mhi_exec(p, t, ip);
4766
4767         /* Thread. */
4768         thread_ip_inc(p);
4769 }
4770
4771 static inline void
4772 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4773 {
4774         struct thread *t = &p->threads[p->thread_id];
4775         struct instruction *ip = t->ip;
4776
4777         /* Structs. */
4778         __instr_meter_mmm_exec(p, t, ip);
4779
4780         /* Thread. */
4781         thread_ip_inc(p);
4782 }
4783
4784 static inline void
4785 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4786 {
4787         struct thread *t = &p->threads[p->thread_id];
4788         struct instruction *ip = t->ip;
4789
4790         /* Structs. */
4791         __instr_meter_mmi_exec(p, t, ip);
4792
4793         /* Thread. */
4794         thread_ip_inc(p);
4795 }
4796
4797 static inline void
4798 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4799 {
4800         struct thread *t = &p->threads[p->thread_id];
4801         struct instruction *ip = t->ip;
4802
4803         /* Structs. */
4804         __instr_meter_ihm_exec(p, t, ip);
4805
4806         /* Thread. */
4807         thread_ip_inc(p);
4808 }
4809
4810 static inline void
4811 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4812 {
4813         struct thread *t = &p->threads[p->thread_id];
4814         struct instruction *ip = t->ip;
4815
4816         /* Structs. */
4817         __instr_meter_ihi_exec(p, t, ip);
4818
4819         /* Thread. */
4820         thread_ip_inc(p);
4821 }
4822
4823 static inline void
4824 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4825 {
4826         struct thread *t = &p->threads[p->thread_id];
4827         struct instruction *ip = t->ip;
4828
4829         /* Structs. */
4830         __instr_meter_imm_exec(p, t, ip);
4831
4832         /* Thread. */
4833         thread_ip_inc(p);
4834 }
4835
4836 static inline void
4837 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4838 {
4839         struct thread *t = &p->threads[p->thread_id];
4840         struct instruction *ip = t->ip;
4841
4842         /* Structs. */
4843         __instr_meter_imi_exec(p, t, ip);
4844
4845         /* Thread. */
4846         thread_ip_inc(p);
4847 }
4848
4849 /*
4850  * jmp.
4851  */
4852 static int
4853 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4854                     struct action *action __rte_unused,
4855                     char **tokens,
4856                     int n_tokens,
4857                     struct instruction *instr,
4858                     struct instruction_data *data)
4859 {
4860         CHECK(n_tokens == 2, EINVAL);
4861
4862         strcpy(data->jmp_label, tokens[1]);
4863
4864         instr->type = INSTR_JMP;
4865         instr->jmp.ip = NULL; /* Resolved later. */
4866         return 0;
4867 }
4868
4869 static int
4870 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4871                           struct action *action __rte_unused,
4872                           char **tokens,
4873                           int n_tokens,
4874                           struct instruction *instr,
4875                           struct instruction_data *data)
4876 {
4877         struct header *h;
4878
4879         CHECK(n_tokens == 3, EINVAL);
4880
4881         strcpy(data->jmp_label, tokens[1]);
4882
4883         h = header_parse(p, tokens[2]);
4884         CHECK(h, EINVAL);
4885
4886         instr->type = INSTR_JMP_VALID;
4887         instr->jmp.ip = NULL; /* Resolved later. */
4888         instr->jmp.header_id = h->id;
4889         return 0;
4890 }
4891
4892 static int
4893 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4894                             struct action *action __rte_unused,
4895                             char **tokens,
4896                             int n_tokens,
4897                             struct instruction *instr,
4898                             struct instruction_data *data)
4899 {
4900         struct header *h;
4901
4902         CHECK(n_tokens == 3, EINVAL);
4903
4904         strcpy(data->jmp_label, tokens[1]);
4905
4906         h = header_parse(p, tokens[2]);
4907         CHECK(h, EINVAL);
4908
4909         instr->type = INSTR_JMP_INVALID;
4910         instr->jmp.ip = NULL; /* Resolved later. */
4911         instr->jmp.header_id = h->id;
4912         return 0;
4913 }
4914
4915 static int
4916 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4917                         struct action *action,
4918                         char **tokens,
4919                         int n_tokens,
4920                         struct instruction *instr,
4921                         struct instruction_data *data)
4922 {
4923         CHECK(!action, EINVAL);
4924         CHECK(n_tokens == 2, EINVAL);
4925
4926         strcpy(data->jmp_label, tokens[1]);
4927
4928         instr->type = INSTR_JMP_HIT;
4929         instr->jmp.ip = NULL; /* Resolved later. */
4930         return 0;
4931 }
4932
4933 static int
4934 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4935                          struct action *action,
4936                          char **tokens,
4937                          int n_tokens,
4938                          struct instruction *instr,
4939                          struct instruction_data *data)
4940 {
4941         CHECK(!action, EINVAL);
4942         CHECK(n_tokens == 2, EINVAL);
4943
4944         strcpy(data->jmp_label, tokens[1]);
4945
4946         instr->type = INSTR_JMP_MISS;
4947         instr->jmp.ip = NULL; /* Resolved later. */
4948         return 0;
4949 }
4950
4951 static int
4952 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4953                                struct action *action,
4954                                char **tokens,
4955                                int n_tokens,
4956                                struct instruction *instr,
4957                                struct instruction_data *data)
4958 {
4959         struct action *a;
4960
4961         CHECK(!action, EINVAL);
4962         CHECK(n_tokens == 3, EINVAL);
4963
4964         strcpy(data->jmp_label, tokens[1]);
4965
4966         a = action_find(p, tokens[2]);
4967         CHECK(a, EINVAL);
4968
4969         instr->type = INSTR_JMP_ACTION_HIT;
4970         instr->jmp.ip = NULL; /* Resolved later. */
4971         instr->jmp.action_id = a->id;
4972         return 0;
4973 }
4974
4975 static int
4976 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4977                                 struct action *action,
4978                                 char **tokens,
4979                                 int n_tokens,
4980                                 struct instruction *instr,
4981                                 struct instruction_data *data)
4982 {
4983         struct action *a;
4984
4985         CHECK(!action, EINVAL);
4986         CHECK(n_tokens == 3, EINVAL);
4987
4988         strcpy(data->jmp_label, tokens[1]);
4989
4990         a = action_find(p, tokens[2]);
4991         CHECK(a, EINVAL);
4992
4993         instr->type = INSTR_JMP_ACTION_MISS;
4994         instr->jmp.ip = NULL; /* Resolved later. */
4995         instr->jmp.action_id = a->id;
4996         return 0;
4997 }
4998
4999 static int
5000 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5001                        struct action *action,
5002                        char **tokens,
5003                        int n_tokens,
5004                        struct instruction *instr,
5005                        struct instruction_data *data)
5006 {
5007         char *a = tokens[2], *b = tokens[3];
5008         struct field *fa, *fb;
5009         uint64_t b_val;
5010         uint32_t a_struct_id, b_struct_id;
5011
5012         CHECK(n_tokens == 4, EINVAL);
5013
5014         strcpy(data->jmp_label, tokens[1]);
5015
5016         fa = struct_field_parse(p, action, a, &a_struct_id);
5017         CHECK(fa, EINVAL);
5018         CHECK(!fa->var_size, EINVAL);
5019
5020         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5021         fb = struct_field_parse(p, action, b, &b_struct_id);
5022         if (fb) {
5023                 CHECK(!fb->var_size, EINVAL);
5024
5025                 instr->type = INSTR_JMP_EQ;
5026                 if (a[0] != 'h' && b[0] == 'h')
5027                         instr->type = INSTR_JMP_EQ_MH;
5028                 if (a[0] == 'h' && b[0] != 'h')
5029                         instr->type = INSTR_JMP_EQ_HM;
5030                 if (a[0] == 'h' && b[0] == 'h')
5031                         instr->type = INSTR_JMP_EQ_HH;
5032                 instr->jmp.ip = NULL; /* Resolved later. */
5033
5034                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5035                 instr->jmp.a.n_bits = fa->n_bits;
5036                 instr->jmp.a.offset = fa->offset / 8;
5037                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5038                 instr->jmp.b.n_bits = fb->n_bits;
5039                 instr->jmp.b.offset = fb->offset / 8;
5040                 return 0;
5041         }
5042
5043         /* JMP_EQ_I. */
5044         b_val = strtoull(b, &b, 0);
5045         CHECK(!b[0], EINVAL);
5046
5047         if (a[0] == 'h')
5048                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5049
5050         instr->type = INSTR_JMP_EQ_I;
5051         instr->jmp.ip = NULL; /* Resolved later. */
5052         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5053         instr->jmp.a.n_bits = fa->n_bits;
5054         instr->jmp.a.offset = fa->offset / 8;
5055         instr->jmp.b_val = b_val;
5056         return 0;
5057 }
5058
5059 static int
5060 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5061                         struct action *action,
5062                         char **tokens,
5063                         int n_tokens,
5064                         struct instruction *instr,
5065                         struct instruction_data *data)
5066 {
5067         char *a = tokens[2], *b = tokens[3];
5068         struct field *fa, *fb;
5069         uint64_t b_val;
5070         uint32_t a_struct_id, b_struct_id;
5071
5072         CHECK(n_tokens == 4, EINVAL);
5073
5074         strcpy(data->jmp_label, tokens[1]);
5075
5076         fa = struct_field_parse(p, action, a, &a_struct_id);
5077         CHECK(fa, EINVAL);
5078         CHECK(!fa->var_size, EINVAL);
5079
5080         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5081         fb = struct_field_parse(p, action, b, &b_struct_id);
5082         if (fb) {
5083                 CHECK(!fb->var_size, EINVAL);
5084
5085                 instr->type = INSTR_JMP_NEQ;
5086                 if (a[0] != 'h' && b[0] == 'h')
5087                         instr->type = INSTR_JMP_NEQ_MH;
5088                 if (a[0] == 'h' && b[0] != 'h')
5089                         instr->type = INSTR_JMP_NEQ_HM;
5090                 if (a[0] == 'h' && b[0] == 'h')
5091                         instr->type = INSTR_JMP_NEQ_HH;
5092                 instr->jmp.ip = NULL; /* Resolved later. */
5093
5094                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5095                 instr->jmp.a.n_bits = fa->n_bits;
5096                 instr->jmp.a.offset = fa->offset / 8;
5097                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5098                 instr->jmp.b.n_bits = fb->n_bits;
5099                 instr->jmp.b.offset = fb->offset / 8;
5100                 return 0;
5101         }
5102
5103         /* JMP_NEQ_I. */
5104         b_val = strtoull(b, &b, 0);
5105         CHECK(!b[0], EINVAL);
5106
5107         if (a[0] == 'h')
5108                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5109
5110         instr->type = INSTR_JMP_NEQ_I;
5111         instr->jmp.ip = NULL; /* Resolved later. */
5112         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5113         instr->jmp.a.n_bits = fa->n_bits;
5114         instr->jmp.a.offset = fa->offset / 8;
5115         instr->jmp.b_val = b_val;
5116         return 0;
5117 }
5118
5119 static int
5120 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5121                        struct action *action,
5122                        char **tokens,
5123                        int n_tokens,
5124                        struct instruction *instr,
5125                        struct instruction_data *data)
5126 {
5127         char *a = tokens[2], *b = tokens[3];
5128         struct field *fa, *fb;
5129         uint64_t b_val;
5130         uint32_t a_struct_id, b_struct_id;
5131
5132         CHECK(n_tokens == 4, EINVAL);
5133
5134         strcpy(data->jmp_label, tokens[1]);
5135
5136         fa = struct_field_parse(p, action, a, &a_struct_id);
5137         CHECK(fa, EINVAL);
5138         CHECK(!fa->var_size, EINVAL);
5139
5140         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5141         fb = struct_field_parse(p, action, b, &b_struct_id);
5142         if (fb) {
5143                 CHECK(!fb->var_size, EINVAL);
5144
5145                 instr->type = INSTR_JMP_LT;
5146                 if (a[0] == 'h' && b[0] != 'h')
5147                         instr->type = INSTR_JMP_LT_HM;
5148                 if (a[0] != 'h' && b[0] == 'h')
5149                         instr->type = INSTR_JMP_LT_MH;
5150                 if (a[0] == 'h' && b[0] == 'h')
5151                         instr->type = INSTR_JMP_LT_HH;
5152                 instr->jmp.ip = NULL; /* Resolved later. */
5153
5154                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5155                 instr->jmp.a.n_bits = fa->n_bits;
5156                 instr->jmp.a.offset = fa->offset / 8;
5157                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5158                 instr->jmp.b.n_bits = fb->n_bits;
5159                 instr->jmp.b.offset = fb->offset / 8;
5160                 return 0;
5161         }
5162
5163         /* JMP_LT_MI, JMP_LT_HI. */
5164         b_val = strtoull(b, &b, 0);
5165         CHECK(!b[0], EINVAL);
5166
5167         instr->type = INSTR_JMP_LT_MI;
5168         if (a[0] == 'h')
5169                 instr->type = INSTR_JMP_LT_HI;
5170         instr->jmp.ip = NULL; /* Resolved later. */
5171
5172         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5173         instr->jmp.a.n_bits = fa->n_bits;
5174         instr->jmp.a.offset = fa->offset / 8;
5175         instr->jmp.b_val = b_val;
5176         return 0;
5177 }
5178
5179 static int
5180 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5181                        struct action *action,
5182                        char **tokens,
5183                        int n_tokens,
5184                        struct instruction *instr,
5185                        struct instruction_data *data)
5186 {
5187         char *a = tokens[2], *b = tokens[3];
5188         struct field *fa, *fb;
5189         uint64_t b_val;
5190         uint32_t a_struct_id, b_struct_id;
5191
5192         CHECK(n_tokens == 4, EINVAL);
5193
5194         strcpy(data->jmp_label, tokens[1]);
5195
5196         fa = struct_field_parse(p, action, a, &a_struct_id);
5197         CHECK(fa, EINVAL);
5198         CHECK(!fa->var_size, EINVAL);
5199
5200         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5201         fb = struct_field_parse(p, action, b, &b_struct_id);
5202         if (fb) {
5203                 CHECK(!fb->var_size, EINVAL);
5204
5205                 instr->type = INSTR_JMP_GT;
5206                 if (a[0] == 'h' && b[0] != 'h')
5207                         instr->type = INSTR_JMP_GT_HM;
5208                 if (a[0] != 'h' && b[0] == 'h')
5209                         instr->type = INSTR_JMP_GT_MH;
5210                 if (a[0] == 'h' && b[0] == 'h')
5211                         instr->type = INSTR_JMP_GT_HH;
5212                 instr->jmp.ip = NULL; /* Resolved later. */
5213
5214                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5215                 instr->jmp.a.n_bits = fa->n_bits;
5216                 instr->jmp.a.offset = fa->offset / 8;
5217                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5218                 instr->jmp.b.n_bits = fb->n_bits;
5219                 instr->jmp.b.offset = fb->offset / 8;
5220                 return 0;
5221         }
5222
5223         /* JMP_GT_MI, JMP_GT_HI. */
5224         b_val = strtoull(b, &b, 0);
5225         CHECK(!b[0], EINVAL);
5226
5227         instr->type = INSTR_JMP_GT_MI;
5228         if (a[0] == 'h')
5229                 instr->type = INSTR_JMP_GT_HI;
5230         instr->jmp.ip = NULL; /* Resolved later. */
5231
5232         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5233         instr->jmp.a.n_bits = fa->n_bits;
5234         instr->jmp.a.offset = fa->offset / 8;
5235         instr->jmp.b_val = b_val;
5236         return 0;
5237 }
5238
5239 static inline void
5240 instr_jmp_exec(struct rte_swx_pipeline *p)
5241 {
5242         struct thread *t = &p->threads[p->thread_id];
5243         struct instruction *ip = t->ip;
5244
5245         TRACE("[Thread %2u] jmp\n", p->thread_id);
5246
5247         thread_ip_set(t, ip->jmp.ip);
5248 }
5249
5250 static inline void
5251 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5252 {
5253         struct thread *t = &p->threads[p->thread_id];
5254         struct instruction *ip = t->ip;
5255         uint32_t header_id = ip->jmp.header_id;
5256
5257         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5258
5259         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5260 }
5261
5262 static inline void
5263 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5264 {
5265         struct thread *t = &p->threads[p->thread_id];
5266         struct instruction *ip = t->ip;
5267         uint32_t header_id = ip->jmp.header_id;
5268
5269         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5270
5271         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5272 }
5273
5274 static inline void
5275 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5276 {
5277         struct thread *t = &p->threads[p->thread_id];
5278         struct instruction *ip = t->ip;
5279         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5280
5281         TRACE("[Thread %2u] jmph\n", p->thread_id);
5282
5283         t->ip = ip_next[t->hit];
5284 }
5285
5286 static inline void
5287 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5288 {
5289         struct thread *t = &p->threads[p->thread_id];
5290         struct instruction *ip = t->ip;
5291         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5292
5293         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5294
5295         t->ip = ip_next[t->hit];
5296 }
5297
5298 static inline void
5299 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5300 {
5301         struct thread *t = &p->threads[p->thread_id];
5302         struct instruction *ip = t->ip;
5303
5304         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5305
5306         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5307 }
5308
5309 static inline void
5310 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5311 {
5312         struct thread *t = &p->threads[p->thread_id];
5313         struct instruction *ip = t->ip;
5314
5315         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5316
5317         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5318 }
5319
5320 static inline void
5321 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5322 {
5323         struct thread *t = &p->threads[p->thread_id];
5324         struct instruction *ip = t->ip;
5325
5326         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5327
5328         JMP_CMP(t, ip, ==);
5329 }
5330
5331 static inline void
5332 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5333 {
5334         struct thread *t = &p->threads[p->thread_id];
5335         struct instruction *ip = t->ip;
5336
5337         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5338
5339         JMP_CMP_MH(t, ip, ==);
5340 }
5341
5342 static inline void
5343 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5344 {
5345         struct thread *t = &p->threads[p->thread_id];
5346         struct instruction *ip = t->ip;
5347
5348         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5349
5350         JMP_CMP_HM(t, ip, ==);
5351 }
5352
5353 static inline void
5354 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5355 {
5356         struct thread *t = &p->threads[p->thread_id];
5357         struct instruction *ip = t->ip;
5358
5359         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5360
5361         JMP_CMP_HH_FAST(t, ip, ==);
5362 }
5363
5364 static inline void
5365 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5366 {
5367         struct thread *t = &p->threads[p->thread_id];
5368         struct instruction *ip = t->ip;
5369
5370         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5371
5372         JMP_CMP_I(t, ip, ==);
5373 }
5374
5375 static inline void
5376 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5377 {
5378         struct thread *t = &p->threads[p->thread_id];
5379         struct instruction *ip = t->ip;
5380
5381         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5382
5383         JMP_CMP(t, ip, !=);
5384 }
5385
5386 static inline void
5387 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5388 {
5389         struct thread *t = &p->threads[p->thread_id];
5390         struct instruction *ip = t->ip;
5391
5392         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5393
5394         JMP_CMP_MH(t, ip, !=);
5395 }
5396
5397 static inline void
5398 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5399 {
5400         struct thread *t = &p->threads[p->thread_id];
5401         struct instruction *ip = t->ip;
5402
5403         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5404
5405         JMP_CMP_HM(t, ip, !=);
5406 }
5407
5408 static inline void
5409 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5410 {
5411         struct thread *t = &p->threads[p->thread_id];
5412         struct instruction *ip = t->ip;
5413
5414         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5415
5416         JMP_CMP_HH_FAST(t, ip, !=);
5417 }
5418
5419 static inline void
5420 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5421 {
5422         struct thread *t = &p->threads[p->thread_id];
5423         struct instruction *ip = t->ip;
5424
5425         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5426
5427         JMP_CMP_I(t, ip, !=);
5428 }
5429
5430 static inline void
5431 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5432 {
5433         struct thread *t = &p->threads[p->thread_id];
5434         struct instruction *ip = t->ip;
5435
5436         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5437
5438         JMP_CMP(t, ip, <);
5439 }
5440
5441 static inline void
5442 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5443 {
5444         struct thread *t = &p->threads[p->thread_id];
5445         struct instruction *ip = t->ip;
5446
5447         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5448
5449         JMP_CMP_MH(t, ip, <);
5450 }
5451
5452 static inline void
5453 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5454 {
5455         struct thread *t = &p->threads[p->thread_id];
5456         struct instruction *ip = t->ip;
5457
5458         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5459
5460         JMP_CMP_HM(t, ip, <);
5461 }
5462
5463 static inline void
5464 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5465 {
5466         struct thread *t = &p->threads[p->thread_id];
5467         struct instruction *ip = t->ip;
5468
5469         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5470
5471         JMP_CMP_HH(t, ip, <);
5472 }
5473
5474 static inline void
5475 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5476 {
5477         struct thread *t = &p->threads[p->thread_id];
5478         struct instruction *ip = t->ip;
5479
5480         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5481
5482         JMP_CMP_MI(t, ip, <);
5483 }
5484
5485 static inline void
5486 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5487 {
5488         struct thread *t = &p->threads[p->thread_id];
5489         struct instruction *ip = t->ip;
5490
5491         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5492
5493         JMP_CMP_HI(t, ip, <);
5494 }
5495
5496 static inline void
5497 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5498 {
5499         struct thread *t = &p->threads[p->thread_id];
5500         struct instruction *ip = t->ip;
5501
5502         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5503
5504         JMP_CMP(t, ip, >);
5505 }
5506
5507 static inline void
5508 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5509 {
5510         struct thread *t = &p->threads[p->thread_id];
5511         struct instruction *ip = t->ip;
5512
5513         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5514
5515         JMP_CMP_MH(t, ip, >);
5516 }
5517
5518 static inline void
5519 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5520 {
5521         struct thread *t = &p->threads[p->thread_id];
5522         struct instruction *ip = t->ip;
5523
5524         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5525
5526         JMP_CMP_HM(t, ip, >);
5527 }
5528
5529 static inline void
5530 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5531 {
5532         struct thread *t = &p->threads[p->thread_id];
5533         struct instruction *ip = t->ip;
5534
5535         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5536
5537         JMP_CMP_HH(t, ip, >);
5538 }
5539
5540 static inline void
5541 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5542 {
5543         struct thread *t = &p->threads[p->thread_id];
5544         struct instruction *ip = t->ip;
5545
5546         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5547
5548         JMP_CMP_MI(t, ip, >);
5549 }
5550
5551 static inline void
5552 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5553 {
5554         struct thread *t = &p->threads[p->thread_id];
5555         struct instruction *ip = t->ip;
5556
5557         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5558
5559         JMP_CMP_HI(t, ip, >);
5560 }
5561
5562 /*
5563  * return.
5564  */
5565 static int
5566 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5567                        struct action *action,
5568                        char **tokens __rte_unused,
5569                        int n_tokens,
5570                        struct instruction *instr,
5571                        struct instruction_data *data __rte_unused)
5572 {
5573         CHECK(action, EINVAL);
5574         CHECK(n_tokens == 1, EINVAL);
5575
5576         instr->type = INSTR_RETURN;
5577         return 0;
5578 }
5579
5580 static inline void
5581 instr_return_exec(struct rte_swx_pipeline *p)
5582 {
5583         struct thread *t = &p->threads[p->thread_id];
5584
5585         TRACE("[Thread %2u] return\n", p->thread_id);
5586
5587         t->ip = t->ret;
5588 }
5589
5590 static int
5591 instr_translate(struct rte_swx_pipeline *p,
5592                 struct action *action,
5593                 char *string,
5594                 struct instruction *instr,
5595                 struct instruction_data *data)
5596 {
5597         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5598         int n_tokens = 0, tpos = 0;
5599
5600         /* Parse the instruction string into tokens. */
5601         for ( ; ; ) {
5602                 char *token;
5603
5604                 token = strtok_r(string, " \t\v", &string);
5605                 if (!token)
5606                         break;
5607
5608                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5609                 CHECK_NAME(token, EINVAL);
5610
5611                 tokens[n_tokens] = token;
5612                 n_tokens++;
5613         }
5614
5615         CHECK(n_tokens, EINVAL);
5616
5617         /* Handle the optional instruction label. */
5618         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5619                 strcpy(data->label, tokens[0]);
5620
5621                 tpos += 2;
5622                 CHECK(n_tokens - tpos, EINVAL);
5623         }
5624
5625         /* Identify the instruction type. */
5626         if (!strcmp(tokens[tpos], "rx"))
5627                 return instr_rx_translate(p,
5628                                           action,
5629                                           &tokens[tpos],
5630                                           n_tokens - tpos,
5631                                           instr,
5632                                           data);
5633
5634         if (!strcmp(tokens[tpos], "tx"))
5635                 return instr_tx_translate(p,
5636                                           action,
5637                                           &tokens[tpos],
5638                                           n_tokens - tpos,
5639                                           instr,
5640                                           data);
5641
5642         if (!strcmp(tokens[tpos], "drop"))
5643                 return instr_drop_translate(p,
5644                                             action,
5645                                             &tokens[tpos],
5646                                             n_tokens - tpos,
5647                                             instr,
5648                                             data);
5649
5650         if (!strcmp(tokens[tpos], "extract"))
5651                 return instr_hdr_extract_translate(p,
5652                                                    action,
5653                                                    &tokens[tpos],
5654                                                    n_tokens - tpos,
5655                                                    instr,
5656                                                    data);
5657
5658         if (!strcmp(tokens[tpos], "lookahead"))
5659                 return instr_hdr_lookahead_translate(p,
5660                                                      action,
5661                                                      &tokens[tpos],
5662                                                      n_tokens - tpos,
5663                                                      instr,
5664                                                      data);
5665
5666         if (!strcmp(tokens[tpos], "emit"))
5667                 return instr_hdr_emit_translate(p,
5668                                                 action,
5669                                                 &tokens[tpos],
5670                                                 n_tokens - tpos,
5671                                                 instr,
5672                                                 data);
5673
5674         if (!strcmp(tokens[tpos], "validate"))
5675                 return instr_hdr_validate_translate(p,
5676                                                     action,
5677                                                     &tokens[tpos],
5678                                                     n_tokens - tpos,
5679                                                     instr,
5680                                                     data);
5681
5682         if (!strcmp(tokens[tpos], "invalidate"))
5683                 return instr_hdr_invalidate_translate(p,
5684                                                       action,
5685                                                       &tokens[tpos],
5686                                                       n_tokens - tpos,
5687                                                       instr,
5688                                                       data);
5689
5690         if (!strcmp(tokens[tpos], "mov"))
5691                 return instr_mov_translate(p,
5692                                            action,
5693                                            &tokens[tpos],
5694                                            n_tokens - tpos,
5695                                            instr,
5696                                            data);
5697
5698         if (!strcmp(tokens[tpos], "add"))
5699                 return instr_alu_add_translate(p,
5700                                                action,
5701                                                &tokens[tpos],
5702                                                n_tokens - tpos,
5703                                                instr,
5704                                                data);
5705
5706         if (!strcmp(tokens[tpos], "sub"))
5707                 return instr_alu_sub_translate(p,
5708                                                action,
5709                                                &tokens[tpos],
5710                                                n_tokens - tpos,
5711                                                instr,
5712                                                data);
5713
5714         if (!strcmp(tokens[tpos], "ckadd"))
5715                 return instr_alu_ckadd_translate(p,
5716                                                  action,
5717                                                  &tokens[tpos],
5718                                                  n_tokens - tpos,
5719                                                  instr,
5720                                                  data);
5721
5722         if (!strcmp(tokens[tpos], "cksub"))
5723                 return instr_alu_cksub_translate(p,
5724                                                  action,
5725                                                  &tokens[tpos],
5726                                                  n_tokens - tpos,
5727                                                  instr,
5728                                                  data);
5729
5730         if (!strcmp(tokens[tpos], "and"))
5731                 return instr_alu_and_translate(p,
5732                                                action,
5733                                                &tokens[tpos],
5734                                                n_tokens - tpos,
5735                                                instr,
5736                                                data);
5737
5738         if (!strcmp(tokens[tpos], "or"))
5739                 return instr_alu_or_translate(p,
5740                                               action,
5741                                               &tokens[tpos],
5742                                               n_tokens - tpos,
5743                                               instr,
5744                                               data);
5745
5746         if (!strcmp(tokens[tpos], "xor"))
5747                 return instr_alu_xor_translate(p,
5748                                                action,
5749                                                &tokens[tpos],
5750                                                n_tokens - tpos,
5751                                                instr,
5752                                                data);
5753
5754         if (!strcmp(tokens[tpos], "shl"))
5755                 return instr_alu_shl_translate(p,
5756                                                action,
5757                                                &tokens[tpos],
5758                                                n_tokens - tpos,
5759                                                instr,
5760                                                data);
5761
5762         if (!strcmp(tokens[tpos], "shr"))
5763                 return instr_alu_shr_translate(p,
5764                                                action,
5765                                                &tokens[tpos],
5766                                                n_tokens - tpos,
5767                                                instr,
5768                                                data);
5769
5770         if (!strcmp(tokens[tpos], "regprefetch"))
5771                 return instr_regprefetch_translate(p,
5772                                                    action,
5773                                                    &tokens[tpos],
5774                                                    n_tokens - tpos,
5775                                                    instr,
5776                                                    data);
5777
5778         if (!strcmp(tokens[tpos], "regrd"))
5779                 return instr_regrd_translate(p,
5780                                              action,
5781                                              &tokens[tpos],
5782                                              n_tokens - tpos,
5783                                              instr,
5784                                              data);
5785
5786         if (!strcmp(tokens[tpos], "regwr"))
5787                 return instr_regwr_translate(p,
5788                                              action,
5789                                              &tokens[tpos],
5790                                              n_tokens - tpos,
5791                                              instr,
5792                                              data);
5793
5794         if (!strcmp(tokens[tpos], "regadd"))
5795                 return instr_regadd_translate(p,
5796                                               action,
5797                                               &tokens[tpos],
5798                                               n_tokens - tpos,
5799                                               instr,
5800                                               data);
5801
5802         if (!strcmp(tokens[tpos], "metprefetch"))
5803                 return instr_metprefetch_translate(p,
5804                                                    action,
5805                                                    &tokens[tpos],
5806                                                    n_tokens - tpos,
5807                                                    instr,
5808                                                    data);
5809
5810         if (!strcmp(tokens[tpos], "meter"))
5811                 return instr_meter_translate(p,
5812                                              action,
5813                                              &tokens[tpos],
5814                                              n_tokens - tpos,
5815                                              instr,
5816                                              data);
5817
5818         if (!strcmp(tokens[tpos], "table"))
5819                 return instr_table_translate(p,
5820                                              action,
5821                                              &tokens[tpos],
5822                                              n_tokens - tpos,
5823                                              instr,
5824                                              data);
5825
5826         if (!strcmp(tokens[tpos], "learn"))
5827                 return instr_learn_translate(p,
5828                                              action,
5829                                              &tokens[tpos],
5830                                              n_tokens - tpos,
5831                                              instr,
5832                                              data);
5833
5834         if (!strcmp(tokens[tpos], "forget"))
5835                 return instr_forget_translate(p,
5836                                               action,
5837                                               &tokens[tpos],
5838                                               n_tokens - tpos,
5839                                               instr,
5840                                               data);
5841
5842         if (!strcmp(tokens[tpos], "extern"))
5843                 return instr_extern_translate(p,
5844                                               action,
5845                                               &tokens[tpos],
5846                                               n_tokens - tpos,
5847                                               instr,
5848                                               data);
5849
5850         if (!strcmp(tokens[tpos], "jmp"))
5851                 return instr_jmp_translate(p,
5852                                            action,
5853                                            &tokens[tpos],
5854                                            n_tokens - tpos,
5855                                            instr,
5856                                            data);
5857
5858         if (!strcmp(tokens[tpos], "jmpv"))
5859                 return instr_jmp_valid_translate(p,
5860                                                  action,
5861                                                  &tokens[tpos],
5862                                                  n_tokens - tpos,
5863                                                  instr,
5864                                                  data);
5865
5866         if (!strcmp(tokens[tpos], "jmpnv"))
5867                 return instr_jmp_invalid_translate(p,
5868                                                    action,
5869                                                    &tokens[tpos],
5870                                                    n_tokens - tpos,
5871                                                    instr,
5872                                                    data);
5873
5874         if (!strcmp(tokens[tpos], "jmph"))
5875                 return instr_jmp_hit_translate(p,
5876                                                action,
5877                                                &tokens[tpos],
5878                                                n_tokens - tpos,
5879                                                instr,
5880                                                data);
5881
5882         if (!strcmp(tokens[tpos], "jmpnh"))
5883                 return instr_jmp_miss_translate(p,
5884                                                 action,
5885                                                 &tokens[tpos],
5886                                                 n_tokens - tpos,
5887                                                 instr,
5888                                                 data);
5889
5890         if (!strcmp(tokens[tpos], "jmpa"))
5891                 return instr_jmp_action_hit_translate(p,
5892                                                       action,
5893                                                       &tokens[tpos],
5894                                                       n_tokens - tpos,
5895                                                       instr,
5896                                                       data);
5897
5898         if (!strcmp(tokens[tpos], "jmpna"))
5899                 return instr_jmp_action_miss_translate(p,
5900                                                        action,
5901                                                        &tokens[tpos],
5902                                                        n_tokens - tpos,
5903                                                        instr,
5904                                                        data);
5905
5906         if (!strcmp(tokens[tpos], "jmpeq"))
5907                 return instr_jmp_eq_translate(p,
5908                                               action,
5909                                               &tokens[tpos],
5910                                               n_tokens - tpos,
5911                                               instr,
5912                                               data);
5913
5914         if (!strcmp(tokens[tpos], "jmpneq"))
5915                 return instr_jmp_neq_translate(p,
5916                                                action,
5917                                                &tokens[tpos],
5918                                                n_tokens - tpos,
5919                                                instr,
5920                                                data);
5921
5922         if (!strcmp(tokens[tpos], "jmplt"))
5923                 return instr_jmp_lt_translate(p,
5924                                               action,
5925                                               &tokens[tpos],
5926                                               n_tokens - tpos,
5927                                               instr,
5928                                               data);
5929
5930         if (!strcmp(tokens[tpos], "jmpgt"))
5931                 return instr_jmp_gt_translate(p,
5932                                               action,
5933                                               &tokens[tpos],
5934                                               n_tokens - tpos,
5935                                               instr,
5936                                               data);
5937
5938         if (!strcmp(tokens[tpos], "return"))
5939                 return instr_return_translate(p,
5940                                               action,
5941                                               &tokens[tpos],
5942                                               n_tokens - tpos,
5943                                               instr,
5944                                               data);
5945
5946         return -EINVAL;
5947 }
5948
5949 static struct instruction_data *
5950 label_find(struct instruction_data *data, uint32_t n, const char *label)
5951 {
5952         uint32_t i;
5953
5954         for (i = 0; i < n; i++)
5955                 if (!strcmp(label, data[i].label))
5956                         return &data[i];
5957
5958         return NULL;
5959 }
5960
5961 static uint32_t
5962 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5963 {
5964         uint32_t count = 0, i;
5965
5966         if (!label[0])
5967                 return 0;
5968
5969         for (i = 0; i < n; i++)
5970                 if (!strcmp(label, data[i].jmp_label))
5971                         count++;
5972
5973         return count;
5974 }
5975
5976 static int
5977 instr_label_check(struct instruction_data *instruction_data,
5978                   uint32_t n_instructions)
5979 {
5980         uint32_t i;
5981
5982         /* Check that all instruction labels are unique. */
5983         for (i = 0; i < n_instructions; i++) {
5984                 struct instruction_data *data = &instruction_data[i];
5985                 char *label = data->label;
5986                 uint32_t j;
5987
5988                 if (!label[0])
5989                         continue;
5990
5991                 for (j = i + 1; j < n_instructions; j++)
5992                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
5993         }
5994
5995         /* Get users for each instruction label. */
5996         for (i = 0; i < n_instructions; i++) {
5997                 struct instruction_data *data = &instruction_data[i];
5998                 char *label = data->label;
5999
6000                 data->n_users = label_is_used(instruction_data,
6001                                               n_instructions,
6002                                               label);
6003         }
6004
6005         return 0;
6006 }
6007
6008 static int
6009 instr_jmp_resolve(struct instruction *instructions,
6010                   struct instruction_data *instruction_data,
6011                   uint32_t n_instructions)
6012 {
6013         uint32_t i;
6014
6015         for (i = 0; i < n_instructions; i++) {
6016                 struct instruction *instr = &instructions[i];
6017                 struct instruction_data *data = &instruction_data[i];
6018                 struct instruction_data *found;
6019
6020                 if (!instruction_is_jmp(instr))
6021                         continue;
6022
6023                 found = label_find(instruction_data,
6024                                    n_instructions,
6025                                    data->jmp_label);
6026                 CHECK(found, EINVAL);
6027
6028                 instr->jmp.ip = &instructions[found - instruction_data];
6029         }
6030
6031         return 0;
6032 }
6033
6034 static int
6035 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6036              struct action *a,
6037              struct instruction *instr,
6038              struct instruction_data *data __rte_unused,
6039              uint32_t n_instructions)
6040 {
6041         if (!a) {
6042                 enum instruction_type type;
6043                 uint32_t i;
6044
6045                 /* Check that the first instruction is rx. */
6046                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6047
6048                 /* Check that there is at least one tx instruction. */
6049                 for (i = 0; i < n_instructions; i++) {
6050                         type = instr[i].type;
6051
6052                         if (instruction_is_tx(type))
6053                                 break;
6054                 }
6055                 CHECK(i < n_instructions, EINVAL);
6056
6057                 /* Check that the last instruction is either tx or unconditional
6058                  * jump.
6059                  */
6060                 type = instr[n_instructions - 1].type;
6061                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6062         }
6063
6064         if (a) {
6065                 enum instruction_type type;
6066                 uint32_t i;
6067
6068                 /* Check that there is at least one return or tx instruction. */
6069                 for (i = 0; i < n_instructions; i++) {
6070                         type = instr[i].type;
6071
6072                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6073                                 break;
6074                 }
6075                 CHECK(i < n_instructions, EINVAL);
6076         }
6077
6078         return 0;
6079 }
6080
6081 static uint32_t
6082 instr_compact(struct instruction *instructions,
6083               struct instruction_data *instruction_data,
6084               uint32_t n_instructions)
6085 {
6086         uint32_t i, pos = 0;
6087
6088         /* Eliminate the invalid instructions that have been optimized out. */
6089         for (i = 0; i < n_instructions; i++) {
6090                 struct instruction *instr = &instructions[i];
6091                 struct instruction_data *data = &instruction_data[i];
6092
6093                 if (data->invalid)
6094                         continue;
6095
6096                 if (i != pos) {
6097                         memcpy(&instructions[pos], instr, sizeof(*instr));
6098                         memcpy(&instruction_data[pos], data, sizeof(*data));
6099                 }
6100
6101                 pos++;
6102         }
6103
6104         return pos;
6105 }
6106
6107 static int
6108 instr_pattern_extract_many_search(struct instruction *instr,
6109                                   struct instruction_data *data,
6110                                   uint32_t n_instr,
6111                                   uint32_t *n_pattern_instr)
6112 {
6113         uint32_t i;
6114
6115         for (i = 0; i < n_instr; i++) {
6116                 if (data[i].invalid)
6117                         break;
6118
6119                 if (instr[i].type != INSTR_HDR_EXTRACT)
6120                         break;
6121
6122                 if (i == RTE_DIM(instr->io.hdr.header_id))
6123                         break;
6124
6125                 if (i && data[i].n_users)
6126                         break;
6127         }
6128
6129         if (i < 2)
6130                 return 0;
6131
6132         *n_pattern_instr = i;
6133         return 1;
6134 }
6135
6136 static void
6137 instr_pattern_extract_many_replace(struct instruction *instr,
6138                                    struct instruction_data *data,
6139                                    uint32_t n_instr)
6140 {
6141         uint32_t i;
6142
6143         for (i = 1; i < n_instr; i++) {
6144                 instr[0].type++;
6145                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6146                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6147                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6148
6149                 data[i].invalid = 1;
6150         }
6151 }
6152
6153 static uint32_t
6154 instr_pattern_extract_many_optimize(struct instruction *instructions,
6155                                     struct instruction_data *instruction_data,
6156                                     uint32_t n_instructions)
6157 {
6158         uint32_t i;
6159
6160         for (i = 0; i < n_instructions; ) {
6161                 struct instruction *instr = &instructions[i];
6162                 struct instruction_data *data = &instruction_data[i];
6163                 uint32_t n_instr = 0;
6164                 int detected;
6165
6166                 /* Extract many. */
6167                 detected = instr_pattern_extract_many_search(instr,
6168                                                              data,
6169                                                              n_instructions - i,
6170                                                              &n_instr);
6171                 if (detected) {
6172                         instr_pattern_extract_many_replace(instr,
6173                                                            data,
6174                                                            n_instr);
6175                         i += n_instr;
6176                         continue;
6177                 }
6178
6179                 /* No pattern starting at the current instruction. */
6180                 i++;
6181         }
6182
6183         /* Eliminate the invalid instructions that have been optimized out. */
6184         n_instructions = instr_compact(instructions,
6185                                        instruction_data,
6186                                        n_instructions);
6187
6188         return n_instructions;
6189 }
6190
6191 static int
6192 instr_pattern_emit_many_tx_search(struct instruction *instr,
6193                                   struct instruction_data *data,
6194                                   uint32_t n_instr,
6195                                   uint32_t *n_pattern_instr)
6196 {
6197         uint32_t i;
6198
6199         for (i = 0; i < n_instr; i++) {
6200                 if (data[i].invalid)
6201                         break;
6202
6203                 if (instr[i].type != INSTR_HDR_EMIT)
6204                         break;
6205
6206                 if (i == RTE_DIM(instr->io.hdr.header_id))
6207                         break;
6208
6209                 if (i && data[i].n_users)
6210                         break;
6211         }
6212
6213         if (!i)
6214                 return 0;
6215
6216         if (instr[i].type != INSTR_TX)
6217                 return 0;
6218
6219         if (data[i].n_users)
6220                 return 0;
6221
6222         i++;
6223
6224         *n_pattern_instr = i;
6225         return 1;
6226 }
6227
6228 static void
6229 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6230                                    struct instruction_data *data,
6231                                    uint32_t n_instr)
6232 {
6233         uint32_t i;
6234
6235         /* Any emit instruction in addition to the first one. */
6236         for (i = 1; i < n_instr - 1; i++) {
6237                 instr[0].type++;
6238                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6239                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6240                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6241
6242                 data[i].invalid = 1;
6243         }
6244
6245         /* The TX instruction is the last one in the pattern. */
6246         instr[0].type++;
6247         instr[0].io.io.offset = instr[i].io.io.offset;
6248         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6249         data[i].invalid = 1;
6250 }
6251
6252 static uint32_t
6253 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6254                                     struct instruction_data *instruction_data,
6255                                     uint32_t n_instructions)
6256 {
6257         uint32_t i;
6258
6259         for (i = 0; i < n_instructions; ) {
6260                 struct instruction *instr = &instructions[i];
6261                 struct instruction_data *data = &instruction_data[i];
6262                 uint32_t n_instr = 0;
6263                 int detected;
6264
6265                 /* Emit many + TX. */
6266                 detected = instr_pattern_emit_many_tx_search(instr,
6267                                                              data,
6268                                                              n_instructions - i,
6269                                                              &n_instr);
6270                 if (detected) {
6271                         instr_pattern_emit_many_tx_replace(instr,
6272                                                            data,
6273                                                            n_instr);
6274                         i += n_instr;
6275                         continue;
6276                 }
6277
6278                 /* No pattern starting at the current instruction. */
6279                 i++;
6280         }
6281
6282         /* Eliminate the invalid instructions that have been optimized out. */
6283         n_instructions = instr_compact(instructions,
6284                                        instruction_data,
6285                                        n_instructions);
6286
6287         return n_instructions;
6288 }
6289
6290 static uint32_t
6291 action_arg_src_mov_count(struct action *a,
6292                          uint32_t arg_id,
6293                          struct instruction *instructions,
6294                          struct instruction_data *instruction_data,
6295                          uint32_t n_instructions);
6296
6297 static int
6298 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6299                                       struct action *a,
6300                                       struct instruction *instr,
6301                                       struct instruction_data *data,
6302                                       uint32_t n_instr,
6303                                       struct instruction *instructions,
6304                                       struct instruction_data *instruction_data,
6305                                       uint32_t n_instructions,
6306                                       uint32_t *n_pattern_instr)
6307 {
6308         struct header *h;
6309         uint32_t src_field_id, i, j;
6310
6311         /* Prerequisites. */
6312         if (!a || !a->st)
6313                 return 0;
6314
6315         /* First instruction: MOV_HM. */
6316         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6317                 return 0;
6318
6319         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6320         if (!h || h->st->var_size)
6321                 return 0;
6322
6323         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6324                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6325                         break;
6326
6327         if (src_field_id == a->st->n_fields)
6328                 return 0;
6329
6330         if (instr[0].mov.dst.offset ||
6331             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6332             instr[0].mov.src.struct_id ||
6333             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6334             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6335                 return 0;
6336
6337         if ((n_instr < h->st->n_fields + 1) ||
6338              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6339                 return 0;
6340
6341         /* Subsequent instructions: MOV_HM. */
6342         for (i = 1; i < h->st->n_fields; i++)
6343                 if (data[i].invalid ||
6344                     data[i].n_users ||
6345                     (instr[i].type != INSTR_MOV_HM) ||
6346                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6347                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6348                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6349                     instr[i].mov.src.struct_id ||
6350                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6351                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6352                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6353                         return 0;
6354
6355         /* Last instruction: HDR_VALIDATE. */
6356         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6357             (instr[i].valid.header_id != h->id))
6358                 return 0;
6359
6360         /* Check that none of the action args that are used as source for this
6361          * DMA transfer are not used as source in any other mov instruction.
6362          */
6363         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6364                 uint32_t n_users;
6365
6366                 n_users = action_arg_src_mov_count(a,
6367                                                    j,
6368                                                    instructions,
6369                                                    instruction_data,
6370                                                    n_instructions);
6371                 if (n_users > 1)
6372                         return 0;
6373         }
6374
6375         *n_pattern_instr = 1 + i;
6376         return 1;
6377 }
6378
6379 static void
6380 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6381                                        struct action *a,
6382                                        struct instruction *instr,
6383                                        struct instruction_data *data,
6384                                        uint32_t n_instr)
6385 {
6386         struct header *h;
6387         uint32_t src_field_id, src_offset, i;
6388
6389         /* Read from the instructions before they are modified. */
6390         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6391         if (!h)
6392                 return;
6393
6394         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6395                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6396                         break;
6397
6398         if (src_field_id == a->st->n_fields)
6399                 return;
6400
6401         src_offset = instr[0].mov.src.offset;
6402
6403         /* Modify the instructions. */
6404         instr[0].type = INSTR_DMA_HT;
6405         instr[0].dma.dst.header_id[0] = h->id;
6406         instr[0].dma.dst.struct_id[0] = h->struct_id;
6407         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6408         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6409
6410         for (i = 1; i < n_instr; i++)
6411                 data[i].invalid = 1;
6412
6413         /* Update the endianness of the action arguments to header endianness. */
6414         for (i = 0; i < h->st->n_fields; i++)
6415                 a->args_endianness[src_field_id + i] = 1;
6416 }
6417
6418 static uint32_t
6419 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6420                                         struct action *a,
6421                                         struct instruction *instructions,
6422                                         struct instruction_data *instruction_data,
6423                                         uint32_t n_instructions)
6424 {
6425         uint32_t i;
6426
6427         if (!a || !a->st)
6428                 return n_instructions;
6429
6430         for (i = 0; i < n_instructions; ) {
6431                 struct instruction *instr = &instructions[i];
6432                 struct instruction_data *data = &instruction_data[i];
6433                 uint32_t n_instr = 0;
6434                 int detected;
6435
6436                 /* Mov all + validate. */
6437                 detected = instr_pattern_mov_all_validate_search(p,
6438                                                                  a,
6439                                                                  instr,
6440                                                                  data,
6441                                                                  n_instructions - i,
6442                                                                  instructions,
6443                                                                  instruction_data,
6444                                                                  n_instructions,
6445                                                                  &n_instr);
6446                 if (detected) {
6447                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6448                         i += n_instr;
6449                         continue;
6450                 }
6451
6452                 /* No pattern starting at the current instruction. */
6453                 i++;
6454         }
6455
6456         /* Eliminate the invalid instructions that have been optimized out. */
6457         n_instructions = instr_compact(instructions,
6458                                        instruction_data,
6459                                        n_instructions);
6460
6461         return n_instructions;
6462 }
6463
6464 static int
6465 instr_pattern_dma_many_search(struct instruction *instr,
6466                               struct instruction_data *data,
6467                               uint32_t n_instr,
6468                               uint32_t *n_pattern_instr)
6469 {
6470         uint32_t i;
6471
6472         for (i = 0; i < n_instr; i++) {
6473                 if (data[i].invalid)
6474                         break;
6475
6476                 if (instr[i].type != INSTR_DMA_HT)
6477                         break;
6478
6479                 if (i == RTE_DIM(instr->dma.dst.header_id))
6480                         break;
6481
6482                 if (i && data[i].n_users)
6483                         break;
6484         }
6485
6486         if (i < 2)
6487                 return 0;
6488
6489         *n_pattern_instr = i;
6490         return 1;
6491 }
6492
6493 static void
6494 instr_pattern_dma_many_replace(struct instruction *instr,
6495                                struct instruction_data *data,
6496                                uint32_t n_instr)
6497 {
6498         uint32_t i;
6499
6500         for (i = 1; i < n_instr; i++) {
6501                 instr[0].type++;
6502                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6503                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6504                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6505                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6506
6507                 data[i].invalid = 1;
6508         }
6509 }
6510
6511 static uint32_t
6512 instr_pattern_dma_many_optimize(struct instruction *instructions,
6513                struct instruction_data *instruction_data,
6514                uint32_t n_instructions)
6515 {
6516         uint32_t i;
6517
6518         for (i = 0; i < n_instructions; ) {
6519                 struct instruction *instr = &instructions[i];
6520                 struct instruction_data *data = &instruction_data[i];
6521                 uint32_t n_instr = 0;
6522                 int detected;
6523
6524                 /* DMA many. */
6525                 detected = instr_pattern_dma_many_search(instr,
6526                                                          data,
6527                                                          n_instructions - i,
6528                                                          &n_instr);
6529                 if (detected) {
6530                         instr_pattern_dma_many_replace(instr, data, n_instr);
6531                         i += n_instr;
6532                         continue;
6533                 }
6534
6535                 /* No pattern starting at the current instruction. */
6536                 i++;
6537         }
6538
6539         /* Eliminate the invalid instructions that have been optimized out. */
6540         n_instructions = instr_compact(instructions,
6541                                        instruction_data,
6542                                        n_instructions);
6543
6544         return n_instructions;
6545 }
6546
6547 static uint32_t
6548 instr_optimize(struct rte_swx_pipeline *p,
6549                struct action *a,
6550                struct instruction *instructions,
6551                struct instruction_data *instruction_data,
6552                uint32_t n_instructions)
6553 {
6554         /* Extract many. */
6555         n_instructions = instr_pattern_extract_many_optimize(instructions,
6556                                                              instruction_data,
6557                                                              n_instructions);
6558
6559         /* Emit many + TX. */
6560         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6561                                                              instruction_data,
6562                                                              n_instructions);
6563
6564         /* Mov all + validate. */
6565         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6566                                                                  a,
6567                                                                  instructions,
6568                                                                  instruction_data,
6569                                                                  n_instructions);
6570
6571         /* DMA many. */
6572         n_instructions = instr_pattern_dma_many_optimize(instructions,
6573                                                          instruction_data,
6574                                                          n_instructions);
6575
6576         return n_instructions;
6577 }
6578
6579 static int
6580 instruction_config(struct rte_swx_pipeline *p,
6581                    struct action *a,
6582                    const char **instructions,
6583                    uint32_t n_instructions)
6584 {
6585         struct instruction *instr = NULL;
6586         struct instruction_data *data = NULL;
6587         int err = 0;
6588         uint32_t i;
6589
6590         CHECK(n_instructions, EINVAL);
6591         CHECK(instructions, EINVAL);
6592         for (i = 0; i < n_instructions; i++)
6593                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6594
6595         /* Memory allocation. */
6596         instr = calloc(n_instructions, sizeof(struct instruction));
6597         if (!instr) {
6598                 err = -ENOMEM;
6599                 goto error;
6600         }
6601
6602         data = calloc(n_instructions, sizeof(struct instruction_data));
6603         if (!data) {
6604                 err = -ENOMEM;
6605                 goto error;
6606         }
6607
6608         for (i = 0; i < n_instructions; i++) {
6609                 char *string = strdup(instructions[i]);
6610                 if (!string) {
6611                         err = -ENOMEM;
6612                         goto error;
6613                 }
6614
6615                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6616                 if (err) {
6617                         free(string);
6618                         goto error;
6619                 }
6620
6621                 free(string);
6622         }
6623
6624         err = instr_label_check(data, n_instructions);
6625         if (err)
6626                 goto error;
6627
6628         err = instr_verify(p, a, instr, data, n_instructions);
6629         if (err)
6630                 goto error;
6631
6632         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6633
6634         err = instr_jmp_resolve(instr, data, n_instructions);
6635         if (err)
6636                 goto error;
6637
6638         if (a) {
6639                 a->instructions = instr;
6640                 a->instruction_data = data;
6641                 a->n_instructions = n_instructions;
6642         } else {
6643                 p->instructions = instr;
6644                 p->instruction_data = data;
6645                 p->n_instructions = n_instructions;
6646         }
6647
6648         return 0;
6649
6650 error:
6651         free(data);
6652         free(instr);
6653         return err;
6654 }
6655
6656 static instr_exec_t instruction_table[] = {
6657         [INSTR_RX] = instr_rx_exec,
6658         [INSTR_TX] = instr_tx_exec,
6659         [INSTR_TX_I] = instr_tx_i_exec,
6660         [INSTR_DROP] = instr_drop_exec,
6661
6662         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6663         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6664         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6665         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6666         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6667         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6668         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6669         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6670         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6671         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6672
6673         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6674         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6675         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6676         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6677         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6678         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6679         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6680         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6681         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6682
6683         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6684         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6685
6686         [INSTR_MOV] = instr_mov_exec,
6687         [INSTR_MOV_MH] = instr_mov_mh_exec,
6688         [INSTR_MOV_HM] = instr_mov_hm_exec,
6689         [INSTR_MOV_HH] = instr_mov_hh_exec,
6690         [INSTR_MOV_I] = instr_mov_i_exec,
6691
6692         [INSTR_DMA_HT] = instr_dma_ht_exec,
6693         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6694         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6695         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6696         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6697         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6698         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6699         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6700
6701         [INSTR_ALU_ADD] = instr_alu_add_exec,
6702         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6703         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6704         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6705         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6706         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6707
6708         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6709         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6710         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6711         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6712         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6713         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6714
6715         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6716         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6717         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6718         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6719
6720         [INSTR_ALU_AND] = instr_alu_and_exec,
6721         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6722         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6723         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6724         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6725
6726         [INSTR_ALU_OR] = instr_alu_or_exec,
6727         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6728         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6729         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6730         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6731
6732         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6733         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6734         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6735         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6736         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6737
6738         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6739         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6740         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6741         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6742         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6743         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6744
6745         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6746         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6747         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6748         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6749         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6750         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6751
6752         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6753         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6754         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6755
6756         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6757         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6758         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6759         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6760         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6761         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6762
6763         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6764         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6765         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6766         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6767         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6768         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6769         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6770         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6771         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6772
6773         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6774         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6775         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6776         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6777         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6778         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6779         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6780         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6781         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6782
6783         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6784         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6785         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6786
6787         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6788         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6789         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6790         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6791         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6792         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6793         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6794         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6795         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6796         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6797         [INSTR_METER_IMM] = instr_meter_imm_exec,
6798         [INSTR_METER_IMI] = instr_meter_imi_exec,
6799
6800         [INSTR_TABLE] = instr_table_exec,
6801         [INSTR_TABLE_AF] = instr_table_af_exec,
6802         [INSTR_SELECTOR] = instr_selector_exec,
6803         [INSTR_LEARNER] = instr_learner_exec,
6804         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6805         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6806         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6807         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6808         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6809
6810         [INSTR_JMP] = instr_jmp_exec,
6811         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6812         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6813         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6814         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6815         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6816         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6817
6818         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6819         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6820         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6821         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6822         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6823
6824         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6825         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6826         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6827         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6828         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6829
6830         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6831         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6832         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6833         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6834         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6835         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6836
6837         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6838         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6839         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6840         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6841         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6842         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6843
6844         [INSTR_RETURN] = instr_return_exec,
6845 };
6846
6847 static int
6848 instruction_table_build(struct rte_swx_pipeline *p)
6849 {
6850         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6851                                       sizeof(struct instr_exec_t *));
6852         if (!p->instruction_table)
6853                 return -EINVAL;
6854
6855         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6856
6857         return 0;
6858 }
6859
6860 static void
6861 instruction_table_build_free(struct rte_swx_pipeline *p)
6862 {
6863         if (!p->instruction_table)
6864                 return;
6865
6866         free(p->instruction_table);
6867         p->instruction_table = NULL;
6868 }
6869
6870 static void
6871 instruction_table_free(struct rte_swx_pipeline *p)
6872 {
6873         instruction_table_build_free(p);
6874 }
6875
6876 static inline void
6877 instr_exec(struct rte_swx_pipeline *p)
6878 {
6879         struct thread *t = &p->threads[p->thread_id];
6880         struct instruction *ip = t->ip;
6881         instr_exec_t instr = p->instruction_table[ip->type];
6882
6883         instr(p);
6884 }
6885
6886 /*
6887  * Action.
6888  */
6889 static struct action *
6890 action_find(struct rte_swx_pipeline *p, const char *name)
6891 {
6892         struct action *elem;
6893
6894         if (!name)
6895                 return NULL;
6896
6897         TAILQ_FOREACH(elem, &p->actions, node)
6898                 if (strcmp(elem->name, name) == 0)
6899                         return elem;
6900
6901         return NULL;
6902 }
6903
6904 static struct action *
6905 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6906 {
6907         struct action *action = NULL;
6908
6909         TAILQ_FOREACH(action, &p->actions, node)
6910                 if (action->id == id)
6911                         return action;
6912
6913         return NULL;
6914 }
6915
6916 static struct field *
6917 action_field_find(struct action *a, const char *name)
6918 {
6919         return a->st ? struct_type_field_find(a->st, name) : NULL;
6920 }
6921
6922 static struct field *
6923 action_field_parse(struct action *action, const char *name)
6924 {
6925         if (name[0] != 't' || name[1] != '.')
6926                 return NULL;
6927
6928         return action_field_find(action, &name[2]);
6929 }
6930
6931 static int
6932 action_has_nbo_args(struct action *a)
6933 {
6934         uint32_t i;
6935
6936         /* Return if the action does not have any args. */
6937         if (!a->st)
6938                 return 0; /* FALSE */
6939
6940         for (i = 0; i < a->st->n_fields; i++)
6941                 if (a->args_endianness[i])
6942                         return 1; /* TRUE */
6943
6944         return 0; /* FALSE */
6945 }
6946
6947 static int
6948 action_does_learning(struct action *a)
6949 {
6950         uint32_t i;
6951
6952         for (i = 0; i < a->n_instructions; i++)
6953                 switch (a->instructions[i].type) {
6954                 case INSTR_LEARNER_LEARN:
6955                         return 1; /* TRUE */
6956
6957                 case INSTR_LEARNER_FORGET:
6958                         return 1; /* TRUE */
6959
6960                 default:
6961                         continue;
6962                 }
6963
6964         return 0; /* FALSE */
6965 }
6966
6967 int
6968 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6969                                const char *name,
6970                                const char *args_struct_type_name,
6971                                const char **instructions,
6972                                uint32_t n_instructions)
6973 {
6974         struct struct_type *args_struct_type = NULL;
6975         struct action *a;
6976         int err;
6977
6978         CHECK(p, EINVAL);
6979
6980         CHECK_NAME(name, EINVAL);
6981         CHECK(!action_find(p, name), EEXIST);
6982
6983         if (args_struct_type_name) {
6984                 CHECK_NAME(args_struct_type_name, EINVAL);
6985                 args_struct_type = struct_type_find(p, args_struct_type_name);
6986                 CHECK(args_struct_type, EINVAL);
6987                 CHECK(!args_struct_type->var_size, EINVAL);
6988         }
6989
6990         /* Node allocation. */
6991         a = calloc(1, sizeof(struct action));
6992         CHECK(a, ENOMEM);
6993         if (args_struct_type) {
6994                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6995                 if (!a->args_endianness) {
6996                         free(a);
6997                         CHECK(0, ENOMEM);
6998                 }
6999         }
7000
7001         /* Node initialization. */
7002         strcpy(a->name, name);
7003         a->st = args_struct_type;
7004         a->id = p->n_actions;
7005
7006         /* Instruction translation. */
7007         err = instruction_config(p, a, instructions, n_instructions);
7008         if (err) {
7009                 free(a->args_endianness);
7010                 free(a);
7011                 return err;
7012         }
7013
7014         /* Node add to tailq. */
7015         TAILQ_INSERT_TAIL(&p->actions, a, node);
7016         p->n_actions++;
7017
7018         return 0;
7019 }
7020
7021 static int
7022 action_build(struct rte_swx_pipeline *p)
7023 {
7024         struct action *action;
7025
7026         /* p->action_instructions. */
7027         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7028         CHECK(p->action_instructions, ENOMEM);
7029
7030         TAILQ_FOREACH(action, &p->actions, node)
7031                 p->action_instructions[action->id] = action->instructions;
7032
7033         /* p->action_funcs. */
7034         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7035         CHECK(p->action_funcs, ENOMEM);
7036
7037         return 0;
7038 }
7039
7040 static void
7041 action_build_free(struct rte_swx_pipeline *p)
7042 {
7043         free(p->action_funcs);
7044         p->action_funcs = NULL;
7045
7046         free(p->action_instructions);
7047         p->action_instructions = NULL;
7048 }
7049
7050 static void
7051 action_free(struct rte_swx_pipeline *p)
7052 {
7053         action_build_free(p);
7054
7055         for ( ; ; ) {
7056                 struct action *action;
7057
7058                 action = TAILQ_FIRST(&p->actions);
7059                 if (!action)
7060                         break;
7061
7062                 TAILQ_REMOVE(&p->actions, action, node);
7063                 free(action->instruction_data);
7064                 free(action->instructions);
7065                 free(action);
7066         }
7067 }
7068
7069 static uint32_t
7070 action_arg_src_mov_count(struct action *a,
7071                          uint32_t arg_id,
7072                          struct instruction *instructions,
7073                          struct instruction_data *instruction_data,
7074                          uint32_t n_instructions)
7075 {
7076         uint32_t offset, n_users = 0, i;
7077
7078         if (!a->st ||
7079             (arg_id >= a->st->n_fields) ||
7080             !instructions ||
7081             !instruction_data ||
7082             !n_instructions)
7083                 return 0;
7084
7085         offset = a->st->fields[arg_id].offset / 8;
7086
7087         for (i = 0; i < n_instructions; i++) {
7088                 struct instruction *instr = &instructions[i];
7089                 struct instruction_data *data = &instruction_data[i];
7090
7091                 if (data->invalid ||
7092                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7093                     instr->mov.src.struct_id ||
7094                     (instr->mov.src.offset != offset))
7095                         continue;
7096
7097                 n_users++;
7098         }
7099
7100         return n_users;
7101 }
7102
7103 /*
7104  * Table.
7105  */
7106 static struct table_type *
7107 table_type_find(struct rte_swx_pipeline *p, const char *name)
7108 {
7109         struct table_type *elem;
7110
7111         TAILQ_FOREACH(elem, &p->table_types, node)
7112                 if (strcmp(elem->name, name) == 0)
7113                         return elem;
7114
7115         return NULL;
7116 }
7117
7118 static struct table_type *
7119 table_type_resolve(struct rte_swx_pipeline *p,
7120                    const char *recommended_type_name,
7121                    enum rte_swx_table_match_type match_type)
7122 {
7123         struct table_type *elem;
7124
7125         /* Only consider the recommended type if the match type is correct. */
7126         if (recommended_type_name)
7127                 TAILQ_FOREACH(elem, &p->table_types, node)
7128                         if (!strcmp(elem->name, recommended_type_name) &&
7129                             (elem->match_type == match_type))
7130                                 return elem;
7131
7132         /* Ignore the recommended type and get the first element with this match
7133          * type.
7134          */
7135         TAILQ_FOREACH(elem, &p->table_types, node)
7136                 if (elem->match_type == match_type)
7137                         return elem;
7138
7139         return NULL;
7140 }
7141
7142 static struct table *
7143 table_find(struct rte_swx_pipeline *p, const char *name)
7144 {
7145         struct table *elem;
7146
7147         TAILQ_FOREACH(elem, &p->tables, node)
7148                 if (strcmp(elem->name, name) == 0)
7149                         return elem;
7150
7151         return NULL;
7152 }
7153
7154 static struct table *
7155 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7156 {
7157         struct table *table = NULL;
7158
7159         TAILQ_FOREACH(table, &p->tables, node)
7160                 if (table->id == id)
7161                         return table;
7162
7163         return NULL;
7164 }
7165
7166 int
7167 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7168                                      const char *name,
7169                                      enum rte_swx_table_match_type match_type,
7170                                      struct rte_swx_table_ops *ops)
7171 {
7172         struct table_type *elem;
7173
7174         CHECK(p, EINVAL);
7175
7176         CHECK_NAME(name, EINVAL);
7177         CHECK(!table_type_find(p, name), EEXIST);
7178
7179         CHECK(ops, EINVAL);
7180         CHECK(ops->create, EINVAL);
7181         CHECK(ops->lkp, EINVAL);
7182         CHECK(ops->free, EINVAL);
7183
7184         /* Node allocation. */
7185         elem = calloc(1, sizeof(struct table_type));
7186         CHECK(elem, ENOMEM);
7187
7188         /* Node initialization. */
7189         strcpy(elem->name, name);
7190         elem->match_type = match_type;
7191         memcpy(&elem->ops, ops, sizeof(*ops));
7192
7193         /* Node add to tailq. */
7194         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7195
7196         return 0;
7197 }
7198
7199 static int
7200 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7201                          uint32_t n_fields,
7202                          enum rte_swx_table_match_type *match_type)
7203 {
7204         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7205
7206         for (i = 0; i < n_fields; i++) {
7207                 struct rte_swx_match_field_params  *f = &fields[i];
7208
7209                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7210                         n_fields_em++;
7211
7212                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7213                         n_fields_lpm++;
7214         }
7215
7216         if ((n_fields_lpm > 1) ||
7217             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7218                 return -EINVAL;
7219
7220         *match_type = (n_fields_em == n_fields) ?
7221                        RTE_SWX_TABLE_MATCH_EXACT :
7222                        RTE_SWX_TABLE_MATCH_WILDCARD;
7223
7224         return 0;
7225 }
7226
7227 static int
7228 table_match_fields_check(struct rte_swx_pipeline *p,
7229                          struct rte_swx_pipeline_table_params *params,
7230                          struct header **header)
7231 {
7232         struct header *h0 = NULL;
7233         struct field *hf, *mf;
7234         uint32_t *offset = NULL, i;
7235         int status = 0;
7236
7237         /* Return if no match fields. */
7238         if (!params->n_fields) {
7239                 if (params->fields) {
7240                         status = -EINVAL;
7241                         goto end;
7242                 }
7243
7244                 if (header)
7245                         *header = NULL;
7246
7247                 return 0;
7248         }
7249
7250         /* Memory allocation. */
7251         offset = calloc(params->n_fields, sizeof(uint32_t));
7252         if (!offset) {
7253                 status = -ENOMEM;
7254                 goto end;
7255         }
7256
7257         /* Check that all the match fields belong to either the same header or
7258          * to the meta-data.
7259          */
7260         hf = header_field_parse(p, params->fields[0].name, &h0);
7261         mf = metadata_field_parse(p, params->fields[0].name);
7262         if ((!hf && !mf) || (hf && hf->var_size)) {
7263                 status = -EINVAL;
7264                 goto end;
7265         }
7266
7267         offset[0] = h0 ? hf->offset : mf->offset;
7268
7269         for (i = 1; i < params->n_fields; i++)
7270                 if (h0) {
7271                         struct header *h;
7272
7273                         hf = header_field_parse(p, params->fields[i].name, &h);
7274                         if (!hf || (h->id != h0->id) || hf->var_size) {
7275                                 status = -EINVAL;
7276                                 goto end;
7277                         }
7278
7279                         offset[i] = hf->offset;
7280                 } else {
7281                         mf = metadata_field_parse(p, params->fields[i].name);
7282                         if (!mf) {
7283                                 status = -EINVAL;
7284                                 goto end;
7285                         }
7286
7287                         offset[i] = mf->offset;
7288                 }
7289
7290         /* Check that there are no duplicated match fields. */
7291         for (i = 0; i < params->n_fields; i++) {
7292                 uint32_t j;
7293
7294                 for (j = 0; j < i; j++)
7295                         if (offset[j] == offset[i]) {
7296                                 status = -EINVAL;
7297                                 goto end;
7298                         }
7299         }
7300
7301         /* Return. */
7302         if (header)
7303                 *header = h0;
7304
7305 end:
7306         free(offset);
7307         return status;
7308 }
7309
7310 int
7311 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7312                               const char *name,
7313                               struct rte_swx_pipeline_table_params *params,
7314                               const char *recommended_table_type_name,
7315                               const char *args,
7316                               uint32_t size)
7317 {
7318         struct table_type *type;
7319         struct table *t = NULL;
7320         struct action *default_action;
7321         struct header *header = NULL;
7322         uint32_t action_data_size_max = 0, i;
7323         int status = 0;
7324
7325         CHECK(p, EINVAL);
7326
7327         CHECK_NAME(name, EINVAL);
7328         CHECK(!table_find(p, name), EEXIST);
7329         CHECK(!selector_find(p, name), EEXIST);
7330         CHECK(!learner_find(p, name), EEXIST);
7331
7332         CHECK(params, EINVAL);
7333
7334         /* Match checks. */
7335         status = table_match_fields_check(p, params, &header);
7336         if (status)
7337                 return status;
7338
7339         /* Action checks. */
7340         CHECK(params->n_actions, EINVAL);
7341         CHECK(params->action_names, EINVAL);
7342         for (i = 0; i < params->n_actions; i++) {
7343                 const char *action_name = params->action_names[i];
7344                 struct action *a;
7345                 uint32_t action_data_size;
7346                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7347
7348                 CHECK_NAME(action_name, EINVAL);
7349
7350                 a = action_find(p, action_name);
7351                 CHECK(a, EINVAL);
7352                 CHECK(!action_does_learning(a), EINVAL);
7353
7354                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7355                 if (action_data_size > action_data_size_max)
7356                         action_data_size_max = action_data_size;
7357
7358                 if (params->action_is_for_table_entries)
7359                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7360                 if (params->action_is_for_default_entry)
7361                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7362                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7363         }
7364
7365         CHECK_NAME(params->default_action_name, EINVAL);
7366         for (i = 0; i < p->n_actions; i++)
7367                 if (!strcmp(params->action_names[i],
7368                             params->default_action_name))
7369                         break;
7370         CHECK(i < params->n_actions, EINVAL);
7371         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7372               EINVAL);
7373
7374         default_action = action_find(p, params->default_action_name);
7375         CHECK((default_action->st && params->default_action_data) ||
7376               !params->default_action_data, EINVAL);
7377
7378         /* Table type checks. */
7379         if (recommended_table_type_name)
7380                 CHECK_NAME(recommended_table_type_name, EINVAL);
7381
7382         if (params->n_fields) {
7383                 enum rte_swx_table_match_type match_type;
7384
7385                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7386                 if (status)
7387                         return status;
7388
7389                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7390                 CHECK(type, EINVAL);
7391         } else {
7392                 type = NULL;
7393         }
7394
7395         /* Memory allocation. */
7396         t = calloc(1, sizeof(struct table));
7397         if (!t)
7398                 goto nomem;
7399
7400         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7401         if (!t->fields)
7402                 goto nomem;
7403
7404         t->actions = calloc(params->n_actions, sizeof(struct action *));
7405         if (!t->actions)
7406                 goto nomem;
7407
7408         if (action_data_size_max) {
7409                 t->default_action_data = calloc(1, action_data_size_max);
7410                 if (!t->default_action_data)
7411                         goto nomem;
7412         }
7413
7414         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7415         if (!t->action_is_for_table_entries)
7416                 goto nomem;
7417
7418         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7419         if (!t->action_is_for_default_entry)
7420                 goto nomem;
7421
7422         /* Node initialization. */
7423         strcpy(t->name, name);
7424         if (args && args[0])
7425                 strcpy(t->args, args);
7426         t->type = type;
7427
7428         for (i = 0; i < params->n_fields; i++) {
7429                 struct rte_swx_match_field_params *field = &params->fields[i];
7430                 struct match_field *f = &t->fields[i];
7431
7432                 f->match_type = field->match_type;
7433                 f->field = header ?
7434                         header_field_parse(p, field->name, NULL) :
7435                         metadata_field_parse(p, field->name);
7436         }
7437         t->n_fields = params->n_fields;
7438         t->header = header;
7439
7440         for (i = 0; i < params->n_actions; i++) {
7441                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7442
7443                 if (params->action_is_for_table_entries)
7444                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7445                 if (params->action_is_for_default_entry)
7446                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7447
7448                 t->actions[i] = action_find(p, params->action_names[i]);
7449                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7450                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7451         }
7452         t->default_action = default_action;
7453         if (default_action->st)
7454                 memcpy(t->default_action_data,
7455                        params->default_action_data,
7456                        default_action->st->n_bits / 8);
7457         t->n_actions = params->n_actions;
7458         t->default_action_is_const = params->default_action_is_const;
7459         t->action_data_size_max = action_data_size_max;
7460
7461         t->size = size;
7462         t->id = p->n_tables;
7463
7464         /* Node add to tailq. */
7465         TAILQ_INSERT_TAIL(&p->tables, t, node);
7466         p->n_tables++;
7467
7468         return 0;
7469
7470 nomem:
7471         if (!t)
7472                 return -ENOMEM;
7473
7474         free(t->action_is_for_default_entry);
7475         free(t->action_is_for_table_entries);
7476         free(t->default_action_data);
7477         free(t->actions);
7478         free(t->fields);
7479         free(t);
7480
7481         return -ENOMEM;
7482 }
7483
7484 static struct rte_swx_table_params *
7485 table_params_get(struct table *table)
7486 {
7487         struct rte_swx_table_params *params;
7488         struct field *first, *last;
7489         uint8_t *key_mask;
7490         uint32_t key_size, key_offset, action_data_size, i;
7491
7492         /* Memory allocation. */
7493         params = calloc(1, sizeof(struct rte_swx_table_params));
7494         if (!params)
7495                 return NULL;
7496
7497         /* Find first (smallest offset) and last (biggest offset) match fields. */
7498         first = table->fields[0].field;
7499         last = table->fields[0].field;
7500
7501         for (i = 0; i < table->n_fields; i++) {
7502                 struct field *f = table->fields[i].field;
7503
7504                 if (f->offset < first->offset)
7505                         first = f;
7506
7507                 if (f->offset > last->offset)
7508                         last = f;
7509         }
7510
7511         /* Key offset and size. */
7512         key_offset = first->offset / 8;
7513         key_size = (last->offset + last->n_bits - first->offset) / 8;
7514
7515         /* Memory allocation. */
7516         key_mask = calloc(1, key_size);
7517         if (!key_mask) {
7518                 free(params);
7519                 return NULL;
7520         }
7521
7522         /* Key mask. */
7523         for (i = 0; i < table->n_fields; i++) {
7524                 struct field *f = table->fields[i].field;
7525                 uint32_t start = (f->offset - first->offset) / 8;
7526                 size_t size = f->n_bits / 8;
7527
7528                 memset(&key_mask[start], 0xFF, size);
7529         }
7530
7531         /* Action data size. */
7532         action_data_size = 0;
7533         for (i = 0; i < table->n_actions; i++) {
7534                 struct action *action = table->actions[i];
7535                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7536
7537                 if (ads > action_data_size)
7538                         action_data_size = ads;
7539         }
7540
7541         /* Fill in. */
7542         params->match_type = table->type->match_type;
7543         params->key_size = key_size;
7544         params->key_offset = key_offset;
7545         params->key_mask0 = key_mask;
7546         params->action_data_size = action_data_size;
7547         params->n_keys_max = table->size;
7548
7549         return params;
7550 }
7551
7552 static void
7553 table_params_free(struct rte_swx_table_params *params)
7554 {
7555         if (!params)
7556                 return;
7557
7558         free(params->key_mask0);
7559         free(params);
7560 }
7561
7562 static int
7563 table_stub_lkp(void *table __rte_unused,
7564                void *mailbox __rte_unused,
7565                uint8_t **key __rte_unused,
7566                uint64_t *action_id __rte_unused,
7567                uint8_t **action_data __rte_unused,
7568                int *hit)
7569 {
7570         *hit = 0;
7571         return 1; /* DONE. */
7572 }
7573
7574 static int
7575 table_build(struct rte_swx_pipeline *p)
7576 {
7577         uint32_t i;
7578
7579         /* Per pipeline: table statistics. */
7580         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7581         CHECK(p->table_stats, ENOMEM);
7582
7583         for (i = 0; i < p->n_tables; i++) {
7584                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7585                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7586         }
7587
7588         /* Per thread: table runt-time. */
7589         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7590                 struct thread *t = &p->threads[i];
7591                 struct table *table;
7592
7593                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7594                 CHECK(t->tables, ENOMEM);
7595
7596                 TAILQ_FOREACH(table, &p->tables, node) {
7597                         struct table_runtime *r = &t->tables[table->id];
7598
7599                         if (table->type) {
7600                                 uint64_t size;
7601
7602                                 size = table->type->ops.mailbox_size_get();
7603
7604                                 /* r->func. */
7605                                 r->func = table->type->ops.lkp;
7606
7607                                 /* r->mailbox. */
7608                                 if (size) {
7609                                         r->mailbox = calloc(1, size);
7610                                         CHECK(r->mailbox, ENOMEM);
7611                                 }
7612
7613                                 /* r->key. */
7614                                 r->key = table->header ?
7615                                         &t->structs[table->header->struct_id] :
7616                                         &t->structs[p->metadata_struct_id];
7617                         } else {
7618                                 r->func = table_stub_lkp;
7619                         }
7620                 }
7621         }
7622
7623         return 0;
7624 }
7625
7626 static void
7627 table_build_free(struct rte_swx_pipeline *p)
7628 {
7629         uint32_t i;
7630
7631         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7632                 struct thread *t = &p->threads[i];
7633                 uint32_t j;
7634
7635                 if (!t->tables)
7636                         continue;
7637
7638                 for (j = 0; j < p->n_tables; j++) {
7639                         struct table_runtime *r = &t->tables[j];
7640
7641                         free(r->mailbox);
7642                 }
7643
7644                 free(t->tables);
7645                 t->tables = NULL;
7646         }
7647
7648         if (p->table_stats) {
7649                 for (i = 0; i < p->n_tables; i++)
7650                         free(p->table_stats[i].n_pkts_action);
7651
7652                 free(p->table_stats);
7653         }
7654 }
7655
7656 static void
7657 table_free(struct rte_swx_pipeline *p)
7658 {
7659         table_build_free(p);
7660
7661         /* Tables. */
7662         for ( ; ; ) {
7663                 struct table *elem;
7664
7665                 elem = TAILQ_FIRST(&p->tables);
7666                 if (!elem)
7667                         break;
7668
7669                 TAILQ_REMOVE(&p->tables, elem, node);
7670                 free(elem->fields);
7671                 free(elem->actions);
7672                 free(elem->default_action_data);
7673                 free(elem);
7674         }
7675
7676         /* Table types. */
7677         for ( ; ; ) {
7678                 struct table_type *elem;
7679
7680                 elem = TAILQ_FIRST(&p->table_types);
7681                 if (!elem)
7682                         break;
7683
7684                 TAILQ_REMOVE(&p->table_types, elem, node);
7685                 free(elem);
7686         }
7687 }
7688
7689 /*
7690  * Selector.
7691  */
7692 static struct selector *
7693 selector_find(struct rte_swx_pipeline *p, const char *name)
7694 {
7695         struct selector *s;
7696
7697         TAILQ_FOREACH(s, &p->selectors, node)
7698                 if (strcmp(s->name, name) == 0)
7699                         return s;
7700
7701         return NULL;
7702 }
7703
7704 static struct selector *
7705 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7706 {
7707         struct selector *s = NULL;
7708
7709         TAILQ_FOREACH(s, &p->selectors, node)
7710                 if (s->id == id)
7711                         return s;
7712
7713         return NULL;
7714 }
7715
7716 static int
7717 selector_fields_check(struct rte_swx_pipeline *p,
7718                       struct rte_swx_pipeline_selector_params *params,
7719                       struct header **header)
7720 {
7721         struct header *h0 = NULL;
7722         struct field *hf, *mf;
7723         uint32_t i;
7724
7725         /* Return if no selector fields. */
7726         if (!params->n_selector_fields || !params->selector_field_names)
7727                 return -EINVAL;
7728
7729         /* Check that all the selector fields either belong to the same header
7730          * or are all meta-data fields.
7731          */
7732         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7733         mf = metadata_field_parse(p, params->selector_field_names[0]);
7734         if (!hf && !mf)
7735                 return -EINVAL;
7736
7737         for (i = 1; i < params->n_selector_fields; i++)
7738                 if (h0) {
7739                         struct header *h;
7740
7741                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7742                         if (!hf || (h->id != h0->id))
7743                                 return -EINVAL;
7744                 } else {
7745                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7746                         if (!mf)
7747                                 return -EINVAL;
7748                 }
7749
7750         /* Check that there are no duplicated match fields. */
7751         for (i = 0; i < params->n_selector_fields; i++) {
7752                 const char *field_name = params->selector_field_names[i];
7753                 uint32_t j;
7754
7755                 for (j = i + 1; j < params->n_selector_fields; j++)
7756                         if (!strcmp(params->selector_field_names[j], field_name))
7757                                 return -EINVAL;
7758         }
7759
7760         /* Return. */
7761         if (header)
7762                 *header = h0;
7763
7764         return 0;
7765 }
7766
7767 int
7768 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7769                                  const char *name,
7770                                  struct rte_swx_pipeline_selector_params *params)
7771 {
7772         struct selector *s;
7773         struct header *selector_header = NULL;
7774         struct field *group_id_field, *member_id_field;
7775         uint32_t i;
7776         int status = 0;
7777
7778         CHECK(p, EINVAL);
7779
7780         CHECK_NAME(name, EINVAL);
7781         CHECK(!table_find(p, name), EEXIST);
7782         CHECK(!selector_find(p, name), EEXIST);
7783         CHECK(!learner_find(p, name), EEXIST);
7784
7785         CHECK(params, EINVAL);
7786
7787         CHECK_NAME(params->group_id_field_name, EINVAL);
7788         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7789         CHECK(group_id_field, EINVAL);
7790
7791         for (i = 0; i < params->n_selector_fields; i++) {
7792                 const char *field_name = params->selector_field_names[i];
7793
7794                 CHECK_NAME(field_name, EINVAL);
7795         }
7796         status = selector_fields_check(p, params, &selector_header);
7797         if (status)
7798                 return status;
7799
7800         CHECK_NAME(params->member_id_field_name, EINVAL);
7801         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7802         CHECK(member_id_field, EINVAL);
7803
7804         CHECK(params->n_groups_max, EINVAL);
7805
7806         CHECK(params->n_members_per_group_max, EINVAL);
7807
7808         /* Memory allocation. */
7809         s = calloc(1, sizeof(struct selector));
7810         if (!s) {
7811                 status = -ENOMEM;
7812                 goto error;
7813         }
7814
7815         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7816         if (!s->selector_fields) {
7817                 status = -ENOMEM;
7818                 goto error;
7819         }
7820
7821         /* Node initialization. */
7822         strcpy(s->name, name);
7823
7824         s->group_id_field = group_id_field;
7825
7826         for (i = 0; i < params->n_selector_fields; i++) {
7827                 const char *field_name = params->selector_field_names[i];
7828
7829                 s->selector_fields[i] = selector_header ?
7830                         header_field_parse(p, field_name, NULL) :
7831                         metadata_field_parse(p, field_name);
7832         }
7833
7834         s->n_selector_fields = params->n_selector_fields;
7835
7836         s->selector_header = selector_header;
7837
7838         s->member_id_field = member_id_field;
7839
7840         s->n_groups_max = params->n_groups_max;
7841
7842         s->n_members_per_group_max = params->n_members_per_group_max;
7843
7844         s->id = p->n_selectors;
7845
7846         /* Node add to tailq. */
7847         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7848         p->n_selectors++;
7849
7850         return 0;
7851
7852 error:
7853         if (!s)
7854                 return status;
7855
7856         free(s->selector_fields);
7857
7858         free(s);
7859
7860         return status;
7861 }
7862
7863 static void
7864 selector_params_free(struct rte_swx_table_selector_params *params)
7865 {
7866         if (!params)
7867                 return;
7868
7869         free(params->selector_mask);
7870
7871         free(params);
7872 }
7873
7874 static struct rte_swx_table_selector_params *
7875 selector_table_params_get(struct selector *s)
7876 {
7877         struct rte_swx_table_selector_params *params = NULL;
7878         struct field *first, *last;
7879         uint32_t i;
7880
7881         /* Memory allocation. */
7882         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7883         if (!params)
7884                 goto error;
7885
7886         /* Group ID. */
7887         params->group_id_offset = s->group_id_field->offset / 8;
7888
7889         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7890         first = s->selector_fields[0];
7891         last = s->selector_fields[0];
7892
7893         for (i = 0; i < s->n_selector_fields; i++) {
7894                 struct field *f = s->selector_fields[i];
7895
7896                 if (f->offset < first->offset)
7897                         first = f;
7898
7899                 if (f->offset > last->offset)
7900                         last = f;
7901         }
7902
7903         /* Selector offset and size. */
7904         params->selector_offset = first->offset / 8;
7905         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7906
7907         /* Memory allocation. */
7908         params->selector_mask = calloc(1, params->selector_size);
7909         if (!params->selector_mask)
7910                 goto error;
7911
7912         /* Selector mask. */
7913         for (i = 0; i < s->n_selector_fields; i++) {
7914                 struct field *f = s->selector_fields[i];
7915                 uint32_t start = (f->offset - first->offset) / 8;
7916                 size_t size = f->n_bits / 8;
7917
7918                 memset(&params->selector_mask[start], 0xFF, size);
7919         }
7920
7921         /* Member ID. */
7922         params->member_id_offset = s->member_id_field->offset / 8;
7923
7924         /* Maximum number of groups. */
7925         params->n_groups_max = s->n_groups_max;
7926
7927         /* Maximum number of members per group. */
7928         params->n_members_per_group_max = s->n_members_per_group_max;
7929
7930         return params;
7931
7932 error:
7933         selector_params_free(params);
7934         return NULL;
7935 }
7936
7937 static void
7938 selector_build_free(struct rte_swx_pipeline *p)
7939 {
7940         uint32_t i;
7941
7942         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7943                 struct thread *t = &p->threads[i];
7944                 uint32_t j;
7945
7946                 if (!t->selectors)
7947                         continue;
7948
7949                 for (j = 0; j < p->n_selectors; j++) {
7950                         struct selector_runtime *r = &t->selectors[j];
7951
7952                         free(r->mailbox);
7953                 }
7954
7955                 free(t->selectors);
7956                 t->selectors = NULL;
7957         }
7958
7959         free(p->selector_stats);
7960         p->selector_stats = NULL;
7961 }
7962
7963 static int
7964 selector_build(struct rte_swx_pipeline *p)
7965 {
7966         uint32_t i;
7967         int status = 0;
7968
7969         /* Per pipeline: selector statistics. */
7970         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7971         if (!p->selector_stats) {
7972                 status = -ENOMEM;
7973                 goto error;
7974         }
7975
7976         /* Per thread: selector run-time. */
7977         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7978                 struct thread *t = &p->threads[i];
7979                 struct selector *s;
7980
7981                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7982                 if (!t->selectors) {
7983                         status = -ENOMEM;
7984                         goto error;
7985                 }
7986
7987                 TAILQ_FOREACH(s, &p->selectors, node) {
7988                         struct selector_runtime *r = &t->selectors[s->id];
7989                         uint64_t size;
7990
7991                         /* r->mailbox. */
7992                         size = rte_swx_table_selector_mailbox_size_get();
7993                         if (size) {
7994                                 r->mailbox = calloc(1, size);
7995                                 if (!r->mailbox) {
7996                                         status = -ENOMEM;
7997                                         goto error;
7998                                 }
7999                         }
8000
8001                         /* r->group_id_buffer. */
8002                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8003
8004                         /* r->selector_buffer. */
8005                         r->selector_buffer = s->selector_header ?
8006                                 &t->structs[s->selector_header->struct_id] :
8007                                 &t->structs[p->metadata_struct_id];
8008
8009                         /* r->member_id_buffer. */
8010                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8011                 }
8012         }
8013
8014         return 0;
8015
8016 error:
8017         selector_build_free(p);
8018         return status;
8019 }
8020
8021 static void
8022 selector_free(struct rte_swx_pipeline *p)
8023 {
8024         selector_build_free(p);
8025
8026         /* Selector tables. */
8027         for ( ; ; ) {
8028                 struct selector *elem;
8029
8030                 elem = TAILQ_FIRST(&p->selectors);
8031                 if (!elem)
8032                         break;
8033
8034                 TAILQ_REMOVE(&p->selectors, elem, node);
8035                 free(elem->selector_fields);
8036                 free(elem);
8037         }
8038 }
8039
8040 /*
8041  * Learner table.
8042  */
8043 static struct learner *
8044 learner_find(struct rte_swx_pipeline *p, const char *name)
8045 {
8046         struct learner *l;
8047
8048         TAILQ_FOREACH(l, &p->learners, node)
8049                 if (!strcmp(l->name, name))
8050                         return l;
8051
8052         return NULL;
8053 }
8054
8055 static struct learner *
8056 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8057 {
8058         struct learner *l = NULL;
8059
8060         TAILQ_FOREACH(l, &p->learners, node)
8061                 if (l->id == id)
8062                         return l;
8063
8064         return NULL;
8065 }
8066
8067 static int
8068 learner_match_fields_check(struct rte_swx_pipeline *p,
8069                            struct rte_swx_pipeline_learner_params *params,
8070                            struct header **header)
8071 {
8072         struct header *h0 = NULL;
8073         struct field *hf, *mf;
8074         uint32_t i;
8075
8076         /* Return if no match fields. */
8077         if (!params->n_fields || !params->field_names)
8078                 return -EINVAL;
8079
8080         /* Check that all the match fields either belong to the same header
8081          * or are all meta-data fields.
8082          */
8083         hf = header_field_parse(p, params->field_names[0], &h0);
8084         mf = metadata_field_parse(p, params->field_names[0]);
8085         if (!hf && !mf)
8086                 return -EINVAL;
8087
8088         for (i = 1; i < params->n_fields; i++)
8089                 if (h0) {
8090                         struct header *h;
8091
8092                         hf = header_field_parse(p, params->field_names[i], &h);
8093                         if (!hf || (h->id != h0->id))
8094                                 return -EINVAL;
8095                 } else {
8096                         mf = metadata_field_parse(p, params->field_names[i]);
8097                         if (!mf)
8098                                 return -EINVAL;
8099                 }
8100
8101         /* Check that there are no duplicated match fields. */
8102         for (i = 0; i < params->n_fields; i++) {
8103                 const char *field_name = params->field_names[i];
8104                 uint32_t j;
8105
8106                 for (j = i + 1; j < params->n_fields; j++)
8107                         if (!strcmp(params->field_names[j], field_name))
8108                                 return -EINVAL;
8109         }
8110
8111         /* Return. */
8112         if (header)
8113                 *header = h0;
8114
8115         return 0;
8116 }
8117
8118 static int
8119 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8120 {
8121         struct struct_type *mst = p->metadata_st, *ast = a->st;
8122         struct field *mf, *af;
8123         uint32_t mf_pos, i;
8124
8125         if (!ast) {
8126                 if (mf_name)
8127                         return -EINVAL;
8128
8129                 return 0;
8130         }
8131
8132         /* Check that mf_name is the name of a valid meta-data field. */
8133         CHECK_NAME(mf_name, EINVAL);
8134         mf = metadata_field_parse(p, mf_name);
8135         CHECK(mf, EINVAL);
8136
8137         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8138          * all the action arguments.
8139          */
8140         mf_pos = mf - mst->fields;
8141         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8142
8143         /* Check that the size of each of the identified meta-data fields matches exactly the size
8144          * of the corresponding action argument.
8145          */
8146         for (i = 0; i < ast->n_fields; i++) {
8147                 mf = &mst->fields[mf_pos + i];
8148                 af = &ast->fields[i];
8149
8150                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8151         }
8152
8153         return 0;
8154 }
8155
8156 static int
8157 learner_action_learning_check(struct rte_swx_pipeline *p,
8158                               struct action *action,
8159                               const char **action_names,
8160                               uint32_t n_actions)
8161 {
8162         uint32_t i;
8163
8164         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8165          * the action passed as argument to the "learn" instruction) is also enabled for the
8166          * current learner table.
8167          */
8168         for (i = 0; i < action->n_instructions; i++) {
8169                 struct instruction *instr = &action->instructions[i];
8170                 uint32_t found = 0, j;
8171
8172                 if (instr->type != INSTR_LEARNER_LEARN)
8173                         continue;
8174
8175                 for (j = 0; j < n_actions; j++) {
8176                         struct action *a;
8177
8178                         a = action_find(p, action_names[j]);
8179                         if (!a)
8180                                 return -EINVAL;
8181
8182                         if (a->id == instr->learn.action_id)
8183                                 found = 1;
8184                 }
8185
8186                 if (!found)
8187                         return -EINVAL;
8188         }
8189
8190         return 0;
8191 }
8192
8193 int
8194 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8195                               const char *name,
8196                               struct rte_swx_pipeline_learner_params *params,
8197                               uint32_t size,
8198                               uint32_t timeout)
8199 {
8200         struct learner *l = NULL;
8201         struct action *default_action;
8202         struct header *header = NULL;
8203         uint32_t action_data_size_max = 0, i;
8204         int status = 0;
8205
8206         CHECK(p, EINVAL);
8207
8208         CHECK_NAME(name, EINVAL);
8209         CHECK(!table_find(p, name), EEXIST);
8210         CHECK(!selector_find(p, name), EEXIST);
8211         CHECK(!learner_find(p, name), EEXIST);
8212
8213         CHECK(params, EINVAL);
8214
8215         /* Match checks. */
8216         status = learner_match_fields_check(p, params, &header);
8217         if (status)
8218                 return status;
8219
8220         /* Action checks. */
8221         CHECK(params->n_actions, EINVAL);
8222         CHECK(params->action_names, EINVAL);
8223         for (i = 0; i < params->n_actions; i++) {
8224                 const char *action_name = params->action_names[i];
8225                 struct action *a;
8226                 uint32_t action_data_size;
8227                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8228
8229                 CHECK_NAME(action_name, EINVAL);
8230
8231                 a = action_find(p, action_name);
8232                 CHECK(a, EINVAL);
8233
8234                 status = learner_action_learning_check(p,
8235                                                        a,
8236                                                        params->action_names,
8237                                                        params->n_actions);
8238                 if (status)
8239                         return status;
8240
8241                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8242                 if (action_data_size > action_data_size_max)
8243                         action_data_size_max = action_data_size;
8244
8245                 if (params->action_is_for_table_entries)
8246                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8247                 if (params->action_is_for_default_entry)
8248                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8249                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8250         }
8251
8252         CHECK_NAME(params->default_action_name, EINVAL);
8253         for (i = 0; i < p->n_actions; i++)
8254                 if (!strcmp(params->action_names[i],
8255                             params->default_action_name))
8256                         break;
8257         CHECK(i < params->n_actions, EINVAL);
8258         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8259               EINVAL);
8260
8261         default_action = action_find(p, params->default_action_name);
8262         CHECK((default_action->st && params->default_action_data) ||
8263               !params->default_action_data, EINVAL);
8264
8265         /* Any other checks. */
8266         CHECK(size, EINVAL);
8267         CHECK(timeout, EINVAL);
8268
8269         /* Memory allocation. */
8270         l = calloc(1, sizeof(struct learner));
8271         if (!l)
8272                 goto nomem;
8273
8274         l->fields = calloc(params->n_fields, sizeof(struct field *));
8275         if (!l->fields)
8276                 goto nomem;
8277
8278         l->actions = calloc(params->n_actions, sizeof(struct action *));
8279         if (!l->actions)
8280                 goto nomem;
8281
8282         if (action_data_size_max) {
8283                 l->default_action_data = calloc(1, action_data_size_max);
8284                 if (!l->default_action_data)
8285                         goto nomem;
8286         }
8287
8288         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8289         if (!l->action_is_for_table_entries)
8290                 goto nomem;
8291
8292         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8293         if (!l->action_is_for_default_entry)
8294                 goto nomem;
8295
8296         /* Node initialization. */
8297         strcpy(l->name, name);
8298
8299         for (i = 0; i < params->n_fields; i++) {
8300                 const char *field_name = params->field_names[i];
8301
8302                 l->fields[i] = header ?
8303                         header_field_parse(p, field_name, NULL) :
8304                         metadata_field_parse(p, field_name);
8305         }
8306
8307         l->n_fields = params->n_fields;
8308
8309         l->header = header;
8310
8311         for (i = 0; i < params->n_actions; i++) {
8312                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8313
8314                 if (params->action_is_for_table_entries)
8315                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8316                 if (params->action_is_for_default_entry)
8317                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8318
8319                 l->actions[i] = action_find(p, params->action_names[i]);
8320                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8321                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8322         }
8323
8324         l->default_action = default_action;
8325
8326         if (default_action->st)
8327                 memcpy(l->default_action_data,
8328                        params->default_action_data,
8329                        default_action->st->n_bits / 8);
8330
8331         l->n_actions = params->n_actions;
8332
8333         l->default_action_is_const = params->default_action_is_const;
8334
8335         l->action_data_size_max = action_data_size_max;
8336
8337         l->size = size;
8338
8339         l->timeout = timeout;
8340
8341         l->id = p->n_learners;
8342
8343         /* Node add to tailq. */
8344         TAILQ_INSERT_TAIL(&p->learners, l, node);
8345         p->n_learners++;
8346
8347         return 0;
8348
8349 nomem:
8350         if (!l)
8351                 return -ENOMEM;
8352
8353         free(l->action_is_for_default_entry);
8354         free(l->action_is_for_table_entries);
8355         free(l->default_action_data);
8356         free(l->actions);
8357         free(l->fields);
8358         free(l);
8359
8360         return -ENOMEM;
8361 }
8362
8363 static void
8364 learner_params_free(struct rte_swx_table_learner_params *params)
8365 {
8366         if (!params)
8367                 return;
8368
8369         free(params->key_mask0);
8370
8371         free(params);
8372 }
8373
8374 static struct rte_swx_table_learner_params *
8375 learner_params_get(struct learner *l)
8376 {
8377         struct rte_swx_table_learner_params *params = NULL;
8378         struct field *first, *last;
8379         uint32_t i;
8380
8381         /* Memory allocation. */
8382         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8383         if (!params)
8384                 goto error;
8385
8386         /* Find first (smallest offset) and last (biggest offset) match fields. */
8387         first = l->fields[0];
8388         last = l->fields[0];
8389
8390         for (i = 0; i < l->n_fields; i++) {
8391                 struct field *f = l->fields[i];
8392
8393                 if (f->offset < first->offset)
8394                         first = f;
8395
8396                 if (f->offset > last->offset)
8397                         last = f;
8398         }
8399
8400         /* Key offset and size. */
8401         params->key_offset = first->offset / 8;
8402         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8403
8404         /* Memory allocation. */
8405         params->key_mask0 = calloc(1, params->key_size);
8406         if (!params->key_mask0)
8407                 goto error;
8408
8409         /* Key mask. */
8410         for (i = 0; i < l->n_fields; i++) {
8411                 struct field *f = l->fields[i];
8412                 uint32_t start = (f->offset - first->offset) / 8;
8413                 size_t size = f->n_bits / 8;
8414
8415                 memset(&params->key_mask0[start], 0xFF, size);
8416         }
8417
8418         /* Action data size. */
8419         params->action_data_size = l->action_data_size_max;
8420
8421         /* Maximum number of keys. */
8422         params->n_keys_max = l->size;
8423
8424         /* Timeout. */
8425         params->key_timeout = l->timeout;
8426
8427         return params;
8428
8429 error:
8430         learner_params_free(params);
8431         return NULL;
8432 }
8433
8434 static void
8435 learner_build_free(struct rte_swx_pipeline *p)
8436 {
8437         uint32_t i;
8438
8439         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8440                 struct thread *t = &p->threads[i];
8441                 uint32_t j;
8442
8443                 if (!t->learners)
8444                         continue;
8445
8446                 for (j = 0; j < p->n_learners; j++) {
8447                         struct learner_runtime *r = &t->learners[j];
8448
8449                         free(r->mailbox);
8450                 }
8451
8452                 free(t->learners);
8453                 t->learners = NULL;
8454         }
8455
8456         if (p->learner_stats) {
8457                 for (i = 0; i < p->n_learners; i++)
8458                         free(p->learner_stats[i].n_pkts_action);
8459
8460                 free(p->learner_stats);
8461         }
8462 }
8463
8464 static int
8465 learner_build(struct rte_swx_pipeline *p)
8466 {
8467         uint32_t i;
8468         int status = 0;
8469
8470         /* Per pipeline: learner statistics. */
8471         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8472         CHECK(p->learner_stats, ENOMEM);
8473
8474         for (i = 0; i < p->n_learners; i++) {
8475                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8476                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8477         }
8478
8479         /* Per thread: learner run-time. */
8480         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8481                 struct thread *t = &p->threads[i];
8482                 struct learner *l;
8483
8484                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8485                 if (!t->learners) {
8486                         status = -ENOMEM;
8487                         goto error;
8488                 }
8489
8490                 TAILQ_FOREACH(l, &p->learners, node) {
8491                         struct learner_runtime *r = &t->learners[l->id];
8492                         uint64_t size;
8493
8494                         /* r->mailbox. */
8495                         size = rte_swx_table_learner_mailbox_size_get();
8496                         if (size) {
8497                                 r->mailbox = calloc(1, size);
8498                                 if (!r->mailbox) {
8499                                         status = -ENOMEM;
8500                                         goto error;
8501                                 }
8502                         }
8503
8504                         /* r->key. */
8505                         r->key = l->header ?
8506                                 &t->structs[l->header->struct_id] :
8507                                 &t->structs[p->metadata_struct_id];
8508                 }
8509         }
8510
8511         return 0;
8512
8513 error:
8514         learner_build_free(p);
8515         return status;
8516 }
8517
8518 static void
8519 learner_free(struct rte_swx_pipeline *p)
8520 {
8521         learner_build_free(p);
8522
8523         /* Learner tables. */
8524         for ( ; ; ) {
8525                 struct learner *l;
8526
8527                 l = TAILQ_FIRST(&p->learners);
8528                 if (!l)
8529                         break;
8530
8531                 TAILQ_REMOVE(&p->learners, l, node);
8532                 free(l->fields);
8533                 free(l->actions);
8534                 free(l->default_action_data);
8535                 free(l);
8536         }
8537 }
8538
8539 /*
8540  * Table state.
8541  */
8542 static int
8543 table_state_build(struct rte_swx_pipeline *p)
8544 {
8545         struct table *table;
8546         struct selector *s;
8547         struct learner *l;
8548
8549         p->table_state = calloc(p->n_tables + p->n_selectors,
8550                                 sizeof(struct rte_swx_table_state));
8551         CHECK(p->table_state, ENOMEM);
8552
8553         TAILQ_FOREACH(table, &p->tables, node) {
8554                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8555
8556                 if (table->type) {
8557                         struct rte_swx_table_params *params;
8558
8559                         /* ts->obj. */
8560                         params = table_params_get(table);
8561                         CHECK(params, ENOMEM);
8562
8563                         ts->obj = table->type->ops.create(params,
8564                                 NULL,
8565                                 table->args,
8566                                 p->numa_node);
8567
8568                         table_params_free(params);
8569                         CHECK(ts->obj, ENODEV);
8570                 }
8571
8572                 /* ts->default_action_data. */
8573                 if (table->action_data_size_max) {
8574                         ts->default_action_data =
8575                                 malloc(table->action_data_size_max);
8576                         CHECK(ts->default_action_data, ENOMEM);
8577
8578                         memcpy(ts->default_action_data,
8579                                table->default_action_data,
8580                                table->action_data_size_max);
8581                 }
8582
8583                 /* ts->default_action_id. */
8584                 ts->default_action_id = table->default_action->id;
8585         }
8586
8587         TAILQ_FOREACH(s, &p->selectors, node) {
8588                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8589                 struct rte_swx_table_selector_params *params;
8590
8591                 /* ts->obj. */
8592                 params = selector_table_params_get(s);
8593                 CHECK(params, ENOMEM);
8594
8595                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8596
8597                 selector_params_free(params);
8598                 CHECK(ts->obj, ENODEV);
8599         }
8600
8601         TAILQ_FOREACH(l, &p->learners, node) {
8602                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8603                         p->n_selectors + l->id];
8604                 struct rte_swx_table_learner_params *params;
8605
8606                 /* ts->obj. */
8607                 params = learner_params_get(l);
8608                 CHECK(params, ENOMEM);
8609
8610                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8611                 learner_params_free(params);
8612                 CHECK(ts->obj, ENODEV);
8613
8614                 /* ts->default_action_data. */
8615                 if (l->action_data_size_max) {
8616                         ts->default_action_data = malloc(l->action_data_size_max);
8617                         CHECK(ts->default_action_data, ENOMEM);
8618
8619                         memcpy(ts->default_action_data,
8620                                l->default_action_data,
8621                                l->action_data_size_max);
8622                 }
8623
8624                 /* ts->default_action_id. */
8625                 ts->default_action_id = l->default_action->id;
8626         }
8627
8628         return 0;
8629 }
8630
8631 static void
8632 table_state_build_free(struct rte_swx_pipeline *p)
8633 {
8634         uint32_t i;
8635
8636         if (!p->table_state)
8637                 return;
8638
8639         for (i = 0; i < p->n_tables; i++) {
8640                 struct rte_swx_table_state *ts = &p->table_state[i];
8641                 struct table *table = table_find_by_id(p, i);
8642
8643                 /* ts->obj. */
8644                 if (table->type && ts->obj)
8645                         table->type->ops.free(ts->obj);
8646
8647                 /* ts->default_action_data. */
8648                 free(ts->default_action_data);
8649         }
8650
8651         for (i = 0; i < p->n_selectors; i++) {
8652                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8653
8654                 /* ts->obj. */
8655                 if (ts->obj)
8656                         rte_swx_table_selector_free(ts->obj);
8657         }
8658
8659         for (i = 0; i < p->n_learners; i++) {
8660                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8661
8662                 /* ts->obj. */
8663                 if (ts->obj)
8664                         rte_swx_table_learner_free(ts->obj);
8665
8666                 /* ts->default_action_data. */
8667                 free(ts->default_action_data);
8668         }
8669
8670         free(p->table_state);
8671         p->table_state = NULL;
8672 }
8673
8674 static void
8675 table_state_free(struct rte_swx_pipeline *p)
8676 {
8677         table_state_build_free(p);
8678 }
8679
8680 /*
8681  * Register array.
8682  */
8683 static struct regarray *
8684 regarray_find(struct rte_swx_pipeline *p, const char *name)
8685 {
8686         struct regarray *elem;
8687
8688         TAILQ_FOREACH(elem, &p->regarrays, node)
8689                 if (!strcmp(elem->name, name))
8690                         return elem;
8691
8692         return NULL;
8693 }
8694
8695 static struct regarray *
8696 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8697 {
8698         struct regarray *elem = NULL;
8699
8700         TAILQ_FOREACH(elem, &p->regarrays, node)
8701                 if (elem->id == id)
8702                         return elem;
8703
8704         return NULL;
8705 }
8706
8707 int
8708 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8709                               const char *name,
8710                               uint32_t size,
8711                               uint64_t init_val)
8712 {
8713         struct regarray *r;
8714
8715         CHECK(p, EINVAL);
8716
8717         CHECK_NAME(name, EINVAL);
8718         CHECK(!regarray_find(p, name), EEXIST);
8719
8720         CHECK(size, EINVAL);
8721         size = rte_align32pow2(size);
8722
8723         /* Memory allocation. */
8724         r = calloc(1, sizeof(struct regarray));
8725         CHECK(r, ENOMEM);
8726
8727         /* Node initialization. */
8728         strcpy(r->name, name);
8729         r->init_val = init_val;
8730         r->size = size;
8731         r->id = p->n_regarrays;
8732
8733         /* Node add to tailq. */
8734         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8735         p->n_regarrays++;
8736
8737         return 0;
8738 }
8739
8740 static int
8741 regarray_build(struct rte_swx_pipeline *p)
8742 {
8743         struct regarray *regarray;
8744
8745         if (!p->n_regarrays)
8746                 return 0;
8747
8748         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8749         CHECK(p->regarray_runtime, ENOMEM);
8750
8751         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8752                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8753                 uint32_t i;
8754
8755                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8756                                          RTE_CACHE_LINE_SIZE,
8757                                          p->numa_node);
8758                 CHECK(r->regarray, ENOMEM);
8759
8760                 if (regarray->init_val)
8761                         for (i = 0; i < regarray->size; i++)
8762                                 r->regarray[i] = regarray->init_val;
8763
8764                 r->size_mask = regarray->size - 1;
8765         }
8766
8767         return 0;
8768 }
8769
8770 static void
8771 regarray_build_free(struct rte_swx_pipeline *p)
8772 {
8773         uint32_t i;
8774
8775         if (!p->regarray_runtime)
8776                 return;
8777
8778         for (i = 0; i < p->n_regarrays; i++) {
8779                 struct regarray *regarray = regarray_find_by_id(p, i);
8780                 struct regarray_runtime *r = &p->regarray_runtime[i];
8781
8782                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8783         }
8784
8785         free(p->regarray_runtime);
8786         p->regarray_runtime = NULL;
8787 }
8788
8789 static void
8790 regarray_free(struct rte_swx_pipeline *p)
8791 {
8792         regarray_build_free(p);
8793
8794         for ( ; ; ) {
8795                 struct regarray *elem;
8796
8797                 elem = TAILQ_FIRST(&p->regarrays);
8798                 if (!elem)
8799                         break;
8800
8801                 TAILQ_REMOVE(&p->regarrays, elem, node);
8802                 free(elem);
8803         }
8804 }
8805
8806 /*
8807  * Meter array.
8808  */
8809 static struct meter_profile *
8810 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8811 {
8812         struct meter_profile *elem;
8813
8814         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8815                 if (!strcmp(elem->name, name))
8816                         return elem;
8817
8818         return NULL;
8819 }
8820
8821 static struct metarray *
8822 metarray_find(struct rte_swx_pipeline *p, const char *name)
8823 {
8824         struct metarray *elem;
8825
8826         TAILQ_FOREACH(elem, &p->metarrays, node)
8827                 if (!strcmp(elem->name, name))
8828                         return elem;
8829
8830         return NULL;
8831 }
8832
8833 static struct metarray *
8834 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8835 {
8836         struct metarray *elem = NULL;
8837
8838         TAILQ_FOREACH(elem, &p->metarrays, node)
8839                 if (elem->id == id)
8840                         return elem;
8841
8842         return NULL;
8843 }
8844
8845 int
8846 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8847                                  const char *name,
8848                                  uint32_t size)
8849 {
8850         struct metarray *m;
8851
8852         CHECK(p, EINVAL);
8853
8854         CHECK_NAME(name, EINVAL);
8855         CHECK(!metarray_find(p, name), EEXIST);
8856
8857         CHECK(size, EINVAL);
8858         size = rte_align32pow2(size);
8859
8860         /* Memory allocation. */
8861         m = calloc(1, sizeof(struct metarray));
8862         CHECK(m, ENOMEM);
8863
8864         /* Node initialization. */
8865         strcpy(m->name, name);
8866         m->size = size;
8867         m->id = p->n_metarrays;
8868
8869         /* Node add to tailq. */
8870         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8871         p->n_metarrays++;
8872
8873         return 0;
8874 }
8875
8876 struct meter_profile meter_profile_default = {
8877         .node = {0},
8878         .name = "",
8879         .params = {0},
8880
8881         .profile = {
8882                 .cbs = 10000,
8883                 .pbs = 10000,
8884                 .cir_period = 1,
8885                 .cir_bytes_per_period = 1,
8886                 .pir_period = 1,
8887                 .pir_bytes_per_period = 1,
8888         },
8889
8890         .n_users = 0,
8891 };
8892
8893 static void
8894 meter_init(struct meter *m)
8895 {
8896         memset(m, 0, sizeof(struct meter));
8897         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8898         m->profile = &meter_profile_default;
8899         m->color_mask = RTE_COLOR_GREEN;
8900
8901         meter_profile_default.n_users++;
8902 }
8903
8904 static int
8905 metarray_build(struct rte_swx_pipeline *p)
8906 {
8907         struct metarray *m;
8908
8909         if (!p->n_metarrays)
8910                 return 0;
8911
8912         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8913         CHECK(p->metarray_runtime, ENOMEM);
8914
8915         TAILQ_FOREACH(m, &p->metarrays, node) {
8916                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8917                 uint32_t i;
8918
8919                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8920                                          RTE_CACHE_LINE_SIZE,
8921                                          p->numa_node);
8922                 CHECK(r->metarray, ENOMEM);
8923
8924                 for (i = 0; i < m->size; i++)
8925                         meter_init(&r->metarray[i]);
8926
8927                 r->size_mask = m->size - 1;
8928         }
8929
8930         return 0;
8931 }
8932
8933 static void
8934 metarray_build_free(struct rte_swx_pipeline *p)
8935 {
8936         uint32_t i;
8937
8938         if (!p->metarray_runtime)
8939                 return;
8940
8941         for (i = 0; i < p->n_metarrays; i++) {
8942                 struct metarray *m = metarray_find_by_id(p, i);
8943                 struct metarray_runtime *r = &p->metarray_runtime[i];
8944
8945                 env_free(r->metarray, m->size * sizeof(struct meter));
8946         }
8947
8948         free(p->metarray_runtime);
8949         p->metarray_runtime = NULL;
8950 }
8951
8952 static void
8953 metarray_free(struct rte_swx_pipeline *p)
8954 {
8955         metarray_build_free(p);
8956
8957         /* Meter arrays. */
8958         for ( ; ; ) {
8959                 struct metarray *elem;
8960
8961                 elem = TAILQ_FIRST(&p->metarrays);
8962                 if (!elem)
8963                         break;
8964
8965                 TAILQ_REMOVE(&p->metarrays, elem, node);
8966                 free(elem);
8967         }
8968
8969         /* Meter profiles. */
8970         for ( ; ; ) {
8971                 struct meter_profile *elem;
8972
8973                 elem = TAILQ_FIRST(&p->meter_profiles);
8974                 if (!elem)
8975                         break;
8976
8977                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8978                 free(elem);
8979         }
8980 }
8981
8982 /*
8983  * Pipeline.
8984  */
8985 int
8986 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8987 {
8988         struct rte_swx_pipeline *pipeline;
8989
8990         /* Check input parameters. */
8991         CHECK(p, EINVAL);
8992
8993         /* Memory allocation. */
8994         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8995         CHECK(pipeline, ENOMEM);
8996
8997         /* Initialization. */
8998         TAILQ_INIT(&pipeline->struct_types);
8999         TAILQ_INIT(&pipeline->port_in_types);
9000         TAILQ_INIT(&pipeline->ports_in);
9001         TAILQ_INIT(&pipeline->port_out_types);
9002         TAILQ_INIT(&pipeline->ports_out);
9003         TAILQ_INIT(&pipeline->extern_types);
9004         TAILQ_INIT(&pipeline->extern_objs);
9005         TAILQ_INIT(&pipeline->extern_funcs);
9006         TAILQ_INIT(&pipeline->headers);
9007         TAILQ_INIT(&pipeline->actions);
9008         TAILQ_INIT(&pipeline->table_types);
9009         TAILQ_INIT(&pipeline->tables);
9010         TAILQ_INIT(&pipeline->selectors);
9011         TAILQ_INIT(&pipeline->learners);
9012         TAILQ_INIT(&pipeline->regarrays);
9013         TAILQ_INIT(&pipeline->meter_profiles);
9014         TAILQ_INIT(&pipeline->metarrays);
9015
9016         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9017         pipeline->numa_node = numa_node;
9018
9019         *p = pipeline;
9020         return 0;
9021 }
9022
9023 void
9024 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9025 {
9026         void *lib;
9027
9028         if (!p)
9029                 return;
9030
9031         lib = p->lib;
9032
9033         free(p->instruction_data);
9034         free(p->instructions);
9035
9036         metarray_free(p);
9037         regarray_free(p);
9038         table_state_free(p);
9039         learner_free(p);
9040         selector_free(p);
9041         table_free(p);
9042         action_free(p);
9043         instruction_table_free(p);
9044         metadata_free(p);
9045         header_free(p);
9046         extern_func_free(p);
9047         extern_obj_free(p);
9048         port_out_free(p);
9049         port_in_free(p);
9050         struct_free(p);
9051
9052         free(p);
9053
9054         if (lib)
9055                 dlclose(lib);
9056 }
9057
9058 int
9059 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9060                                      const char **instructions,
9061                                      uint32_t n_instructions)
9062 {
9063         int err;
9064         uint32_t i;
9065
9066         err = instruction_config(p, NULL, instructions, n_instructions);
9067         if (err)
9068                 return err;
9069
9070         /* Thread instruction pointer reset. */
9071         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9072                 struct thread *t = &p->threads[i];
9073
9074                 thread_ip_reset(p, t);
9075         }
9076
9077         return 0;
9078 }
9079
9080 static int
9081 pipeline_compile(struct rte_swx_pipeline *p);
9082
9083 int
9084 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9085 {
9086         int status;
9087
9088         CHECK(p, EINVAL);
9089         CHECK(p->build_done == 0, EEXIST);
9090
9091         status = port_in_build(p);
9092         if (status)
9093                 goto error;
9094
9095         status = port_out_build(p);
9096         if (status)
9097                 goto error;
9098
9099         status = struct_build(p);
9100         if (status)
9101                 goto error;
9102
9103         status = extern_obj_build(p);
9104         if (status)
9105                 goto error;
9106
9107         status = extern_func_build(p);
9108         if (status)
9109                 goto error;
9110
9111         status = header_build(p);
9112         if (status)
9113                 goto error;
9114
9115         status = metadata_build(p);
9116         if (status)
9117                 goto error;
9118
9119         status = instruction_table_build(p);
9120         if (status)
9121                 goto error;
9122
9123         status = action_build(p);
9124         if (status)
9125                 goto error;
9126
9127         status = table_build(p);
9128         if (status)
9129                 goto error;
9130
9131         status = selector_build(p);
9132         if (status)
9133                 goto error;
9134
9135         status = learner_build(p);
9136         if (status)
9137                 goto error;
9138
9139         status = table_state_build(p);
9140         if (status)
9141                 goto error;
9142
9143         status = regarray_build(p);
9144         if (status)
9145                 goto error;
9146
9147         status = metarray_build(p);
9148         if (status)
9149                 goto error;
9150
9151         p->build_done = 1;
9152
9153         pipeline_compile(p);
9154
9155         return 0;
9156
9157 error:
9158         metarray_build_free(p);
9159         regarray_build_free(p);
9160         table_state_build_free(p);
9161         learner_build_free(p);
9162         selector_build_free(p);
9163         table_build_free(p);
9164         action_build_free(p);
9165         instruction_table_build_free(p);
9166         metadata_build_free(p);
9167         header_build_free(p);
9168         extern_func_build_free(p);
9169         extern_obj_build_free(p);
9170         port_out_build_free(p);
9171         port_in_build_free(p);
9172         struct_build_free(p);
9173
9174         return status;
9175 }
9176
9177 void
9178 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9179 {
9180         uint32_t i;
9181
9182         for (i = 0; i < n_instructions; i++)
9183                 instr_exec(p);
9184 }
9185
9186 void
9187 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9188 {
9189         uint32_t i;
9190
9191         for (i = 0; i < p->n_ports_out; i++) {
9192                 struct port_out_runtime *port = &p->out[i];
9193
9194                 if (port->flush)
9195                         port->flush(port->obj);
9196         }
9197 }
9198
9199 /*
9200  * Control.
9201  */
9202 int
9203 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9204                               struct rte_swx_ctl_pipeline_info *pipeline)
9205 {
9206         struct action *action;
9207         struct table *table;
9208         uint32_t n_actions = 0, n_tables = 0;
9209
9210         if (!p || !pipeline)
9211                 return -EINVAL;
9212
9213         TAILQ_FOREACH(action, &p->actions, node)
9214                 n_actions++;
9215
9216         TAILQ_FOREACH(table, &p->tables, node)
9217                 n_tables++;
9218
9219         pipeline->n_ports_in = p->n_ports_in;
9220         pipeline->n_ports_out = p->n_ports_out;
9221         pipeline->n_actions = n_actions;
9222         pipeline->n_tables = n_tables;
9223         pipeline->n_selectors = p->n_selectors;
9224         pipeline->n_learners = p->n_learners;
9225         pipeline->n_regarrays = p->n_regarrays;
9226         pipeline->n_metarrays = p->n_metarrays;
9227
9228         return 0;
9229 }
9230
9231 int
9232 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9233 {
9234         if (!p || !numa_node)
9235                 return -EINVAL;
9236
9237         *numa_node = p->numa_node;
9238         return 0;
9239 }
9240
9241 int
9242 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9243                             uint32_t action_id,
9244                             struct rte_swx_ctl_action_info *action)
9245 {
9246         struct action *a = NULL;
9247
9248         if (!p || (action_id >= p->n_actions) || !action)
9249                 return -EINVAL;
9250
9251         a = action_find_by_id(p, action_id);
9252         if (!a)
9253                 return -EINVAL;
9254
9255         strcpy(action->name, a->name);
9256         action->n_args = a->st ? a->st->n_fields : 0;
9257         return 0;
9258 }
9259
9260 int
9261 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9262                                 uint32_t action_id,
9263                                 uint32_t action_arg_id,
9264                                 struct rte_swx_ctl_action_arg_info *action_arg)
9265 {
9266         struct action *a = NULL;
9267         struct field *arg = NULL;
9268
9269         if (!p || (action_id >= p->n_actions) || !action_arg)
9270                 return -EINVAL;
9271
9272         a = action_find_by_id(p, action_id);
9273         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9274                 return -EINVAL;
9275
9276         arg = &a->st->fields[action_arg_id];
9277         strcpy(action_arg->name, arg->name);
9278         action_arg->n_bits = arg->n_bits;
9279         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9280
9281         return 0;
9282 }
9283
9284 int
9285 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9286                            uint32_t table_id,
9287                            struct rte_swx_ctl_table_info *table)
9288 {
9289         struct table *t = NULL;
9290
9291         if (!p || !table)
9292                 return -EINVAL;
9293
9294         t = table_find_by_id(p, table_id);
9295         if (!t)
9296                 return -EINVAL;
9297
9298         strcpy(table->name, t->name);
9299         strcpy(table->args, t->args);
9300         table->n_match_fields = t->n_fields;
9301         table->n_actions = t->n_actions;
9302         table->default_action_is_const = t->default_action_is_const;
9303         table->size = t->size;
9304         return 0;
9305 }
9306
9307 int
9308 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9309         uint32_t table_id,
9310         uint32_t match_field_id,
9311         struct rte_swx_ctl_table_match_field_info *match_field)
9312 {
9313         struct table *t;
9314         struct match_field *f;
9315
9316         if (!p || (table_id >= p->n_tables) || !match_field)
9317                 return -EINVAL;
9318
9319         t = table_find_by_id(p, table_id);
9320         if (!t || (match_field_id >= t->n_fields))
9321                 return -EINVAL;
9322
9323         f = &t->fields[match_field_id];
9324         match_field->match_type = f->match_type;
9325         match_field->is_header = t->header ? 1 : 0;
9326         match_field->n_bits = f->field->n_bits;
9327         match_field->offset = f->field->offset;
9328
9329         return 0;
9330 }
9331
9332 int
9333 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9334         uint32_t table_id,
9335         uint32_t table_action_id,
9336         struct rte_swx_ctl_table_action_info *table_action)
9337 {
9338         struct table *t;
9339
9340         if (!p || (table_id >= p->n_tables) || !table_action)
9341                 return -EINVAL;
9342
9343         t = table_find_by_id(p, table_id);
9344         if (!t || (table_action_id >= t->n_actions))
9345                 return -EINVAL;
9346
9347         table_action->action_id = t->actions[table_action_id]->id;
9348
9349         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9350         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9351
9352         return 0;
9353 }
9354
9355 int
9356 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9357                           uint32_t table_id,
9358                           struct rte_swx_table_ops *table_ops,
9359                           int *is_stub)
9360 {
9361         struct table *t;
9362
9363         if (!p || (table_id >= p->n_tables))
9364                 return -EINVAL;
9365
9366         t = table_find_by_id(p, table_id);
9367         if (!t)
9368                 return -EINVAL;
9369
9370         if (t->type) {
9371                 if (table_ops)
9372                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9373                 *is_stub = 0;
9374         } else {
9375                 *is_stub = 1;
9376         }
9377
9378         return 0;
9379 }
9380
9381 int
9382 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9383                               uint32_t selector_id,
9384                               struct rte_swx_ctl_selector_info *selector)
9385 {
9386         struct selector *s = NULL;
9387
9388         if (!p || !selector)
9389                 return -EINVAL;
9390
9391         s = selector_find_by_id(p, selector_id);
9392         if (!s)
9393                 return -EINVAL;
9394
9395         strcpy(selector->name, s->name);
9396
9397         selector->n_selector_fields = s->n_selector_fields;
9398         selector->n_groups_max = s->n_groups_max;
9399         selector->n_members_per_group_max = s->n_members_per_group_max;
9400
9401         return 0;
9402 }
9403
9404 int
9405 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9406          uint32_t selector_id,
9407          struct rte_swx_ctl_table_match_field_info *field)
9408 {
9409         struct selector *s;
9410
9411         if (!p || (selector_id >= p->n_selectors) || !field)
9412                 return -EINVAL;
9413
9414         s = selector_find_by_id(p, selector_id);
9415         if (!s)
9416                 return -EINVAL;
9417
9418         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9419         field->is_header = 0;
9420         field->n_bits = s->group_id_field->n_bits;
9421         field->offset = s->group_id_field->offset;
9422
9423         return 0;
9424 }
9425
9426 int
9427 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9428          uint32_t selector_id,
9429          uint32_t selector_field_id,
9430          struct rte_swx_ctl_table_match_field_info *field)
9431 {
9432         struct selector *s;
9433         struct field *f;
9434
9435         if (!p || (selector_id >= p->n_selectors) || !field)
9436                 return -EINVAL;
9437
9438         s = selector_find_by_id(p, selector_id);
9439         if (!s || (selector_field_id >= s->n_selector_fields))
9440                 return -EINVAL;
9441
9442         f = s->selector_fields[selector_field_id];
9443         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9444         field->is_header = s->selector_header ? 1 : 0;
9445         field->n_bits = f->n_bits;
9446         field->offset = f->offset;
9447
9448         return 0;
9449 }
9450
9451 int
9452 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9453          uint32_t selector_id,
9454          struct rte_swx_ctl_table_match_field_info *field)
9455 {
9456         struct selector *s;
9457
9458         if (!p || (selector_id >= p->n_selectors) || !field)
9459                 return -EINVAL;
9460
9461         s = selector_find_by_id(p, selector_id);
9462         if (!s)
9463                 return -EINVAL;
9464
9465         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9466         field->is_header = 0;
9467         field->n_bits = s->member_id_field->n_bits;
9468         field->offset = s->member_id_field->offset;
9469
9470         return 0;
9471 }
9472
9473 int
9474 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9475                              uint32_t learner_id,
9476                              struct rte_swx_ctl_learner_info *learner)
9477 {
9478         struct learner *l = NULL;
9479
9480         if (!p || !learner)
9481                 return -EINVAL;
9482
9483         l = learner_find_by_id(p, learner_id);
9484         if (!l)
9485                 return -EINVAL;
9486
9487         strcpy(learner->name, l->name);
9488
9489         learner->n_match_fields = l->n_fields;
9490         learner->n_actions = l->n_actions;
9491         learner->default_action_is_const = l->default_action_is_const;
9492         learner->size = l->size;
9493
9494         return 0;
9495 }
9496
9497 int
9498 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9499                                          uint32_t learner_id,
9500                                          uint32_t match_field_id,
9501                                          struct rte_swx_ctl_table_match_field_info *match_field)
9502 {
9503         struct learner *l;
9504         struct field *f;
9505
9506         if (!p || (learner_id >= p->n_learners) || !match_field)
9507                 return -EINVAL;
9508
9509         l = learner_find_by_id(p, learner_id);
9510         if (!l || (match_field_id >= l->n_fields))
9511                 return -EINVAL;
9512
9513         f = l->fields[match_field_id];
9514         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9515         match_field->is_header = l->header ? 1 : 0;
9516         match_field->n_bits = f->n_bits;
9517         match_field->offset = f->offset;
9518
9519         return 0;
9520 }
9521
9522 int
9523 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9524                                     uint32_t learner_id,
9525                                     uint32_t learner_action_id,
9526                                     struct rte_swx_ctl_table_action_info *learner_action)
9527 {
9528         struct learner *l;
9529
9530         if (!p || (learner_id >= p->n_learners) || !learner_action)
9531                 return -EINVAL;
9532
9533         l = learner_find_by_id(p, learner_id);
9534         if (!l || (learner_action_id >= l->n_actions))
9535                 return -EINVAL;
9536
9537         learner_action->action_id = l->actions[learner_action_id]->id;
9538
9539         learner_action->action_is_for_table_entries =
9540                 l->action_is_for_table_entries[learner_action_id];
9541
9542         learner_action->action_is_for_default_entry =
9543                 l->action_is_for_default_entry[learner_action_id];
9544
9545         return 0;
9546 }
9547
9548 int
9549 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9550                                  struct rte_swx_table_state **table_state)
9551 {
9552         if (!p || !table_state || !p->build_done)
9553                 return -EINVAL;
9554
9555         *table_state = p->table_state;
9556         return 0;
9557 }
9558
9559 int
9560 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9561                                  struct rte_swx_table_state *table_state)
9562 {
9563         if (!p || !table_state || !p->build_done)
9564                 return -EINVAL;
9565
9566         p->table_state = table_state;
9567         return 0;
9568 }
9569
9570 int
9571 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9572                                         uint32_t port_id,
9573                                         struct rte_swx_port_in_stats *stats)
9574 {
9575         struct port_in *port;
9576
9577         if (!p || !stats)
9578                 return -EINVAL;
9579
9580         port = port_in_find(p, port_id);
9581         if (!port)
9582                 return -EINVAL;
9583
9584         port->type->ops.stats_read(port->obj, stats);
9585         return 0;
9586 }
9587
9588 int
9589 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9590                                          uint32_t port_id,
9591                                          struct rte_swx_port_out_stats *stats)
9592 {
9593         struct port_out *port;
9594
9595         if (!p || !stats)
9596                 return -EINVAL;
9597
9598         port = port_out_find(p, port_id);
9599         if (!port)
9600                 return -EINVAL;
9601
9602         port->type->ops.stats_read(port->obj, stats);
9603         return 0;
9604 }
9605
9606 int
9607 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9608                                       const char *table_name,
9609                                       struct rte_swx_table_stats *stats)
9610 {
9611         struct table *table;
9612         struct table_statistics *table_stats;
9613
9614         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9615                 return -EINVAL;
9616
9617         table = table_find(p, table_name);
9618         if (!table)
9619                 return -EINVAL;
9620
9621         table_stats = &p->table_stats[table->id];
9622
9623         memcpy(stats->n_pkts_action,
9624                table_stats->n_pkts_action,
9625                p->n_actions * sizeof(uint64_t));
9626
9627         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9628         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9629
9630         return 0;
9631 }
9632
9633 int
9634 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9635         const char *selector_name,
9636         struct rte_swx_pipeline_selector_stats *stats)
9637 {
9638         struct selector *s;
9639
9640         if (!p || !selector_name || !selector_name[0] || !stats)
9641                 return -EINVAL;
9642
9643         s = selector_find(p, selector_name);
9644         if (!s)
9645                 return -EINVAL;
9646
9647         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9648
9649         return 0;
9650 }
9651
9652 int
9653 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9654                                         const char *learner_name,
9655                                         struct rte_swx_learner_stats *stats)
9656 {
9657         struct learner *l;
9658         struct learner_statistics *learner_stats;
9659
9660         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9661                 return -EINVAL;
9662
9663         l = learner_find(p, learner_name);
9664         if (!l)
9665                 return -EINVAL;
9666
9667         learner_stats = &p->learner_stats[l->id];
9668
9669         memcpy(stats->n_pkts_action,
9670                learner_stats->n_pkts_action,
9671                p->n_actions * sizeof(uint64_t));
9672
9673         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9674         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9675
9676         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9677         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9678
9679         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9680
9681         return 0;
9682 }
9683
9684 int
9685 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9686                               uint32_t regarray_id,
9687                               struct rte_swx_ctl_regarray_info *regarray)
9688 {
9689         struct regarray *r;
9690
9691         if (!p || !regarray)
9692                 return -EINVAL;
9693
9694         r = regarray_find_by_id(p, regarray_id);
9695         if (!r)
9696                 return -EINVAL;
9697
9698         strcpy(regarray->name, r->name);
9699         regarray->size = r->size;
9700         return 0;
9701 }
9702
9703 int
9704 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9705                                    const char *regarray_name,
9706                                    uint32_t regarray_index,
9707                                    uint64_t *value)
9708 {
9709         struct regarray *regarray;
9710         struct regarray_runtime *r;
9711
9712         if (!p || !regarray_name || !value)
9713                 return -EINVAL;
9714
9715         regarray = regarray_find(p, regarray_name);
9716         if (!regarray || (regarray_index >= regarray->size))
9717                 return -EINVAL;
9718
9719         r = &p->regarray_runtime[regarray->id];
9720         *value = r->regarray[regarray_index];
9721         return 0;
9722 }
9723
9724 int
9725 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9726                                    const char *regarray_name,
9727                                    uint32_t regarray_index,
9728                                    uint64_t value)
9729 {
9730         struct regarray *regarray;
9731         struct regarray_runtime *r;
9732
9733         if (!p || !regarray_name)
9734                 return -EINVAL;
9735
9736         regarray = regarray_find(p, regarray_name);
9737         if (!regarray || (regarray_index >= regarray->size))
9738                 return -EINVAL;
9739
9740         r = &p->regarray_runtime[regarray->id];
9741         r->regarray[regarray_index] = value;
9742         return 0;
9743 }
9744
9745 int
9746 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9747                               uint32_t metarray_id,
9748                               struct rte_swx_ctl_metarray_info *metarray)
9749 {
9750         struct metarray *m;
9751
9752         if (!p || !metarray)
9753                 return -EINVAL;
9754
9755         m = metarray_find_by_id(p, metarray_id);
9756         if (!m)
9757                 return -EINVAL;
9758
9759         strcpy(metarray->name, m->name);
9760         metarray->size = m->size;
9761         return 0;
9762 }
9763
9764 int
9765 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9766                               const char *name,
9767                               struct rte_meter_trtcm_params *params)
9768 {
9769         struct meter_profile *mp;
9770         int status;
9771
9772         CHECK(p, EINVAL);
9773         CHECK_NAME(name, EINVAL);
9774         CHECK(params, EINVAL);
9775         CHECK(!meter_profile_find(p, name), EEXIST);
9776
9777         /* Node allocation. */
9778         mp = calloc(1, sizeof(struct meter_profile));
9779         CHECK(mp, ENOMEM);
9780
9781         /* Node initialization. */
9782         strcpy(mp->name, name);
9783         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9784         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9785         if (status) {
9786                 free(mp);
9787                 CHECK(0, EINVAL);
9788         }
9789
9790         /* Node add to tailq. */
9791         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9792
9793         return 0;
9794 }
9795
9796 int
9797 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9798                                  const char *name)
9799 {
9800         struct meter_profile *mp;
9801
9802         CHECK(p, EINVAL);
9803         CHECK_NAME(name, EINVAL);
9804
9805         mp = meter_profile_find(p, name);
9806         CHECK(mp, EINVAL);
9807         CHECK(!mp->n_users, EBUSY);
9808
9809         /* Remove node from tailq. */
9810         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9811         free(mp);
9812
9813         return 0;
9814 }
9815
9816 int
9817 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9818                         const char *metarray_name,
9819                         uint32_t metarray_index)
9820 {
9821         struct meter_profile *mp_old;
9822         struct metarray *metarray;
9823         struct metarray_runtime *metarray_runtime;
9824         struct meter *m;
9825
9826         CHECK(p, EINVAL);
9827         CHECK_NAME(metarray_name, EINVAL);
9828
9829         metarray = metarray_find(p, metarray_name);
9830         CHECK(metarray, EINVAL);
9831         CHECK(metarray_index < metarray->size, EINVAL);
9832
9833         metarray_runtime = &p->metarray_runtime[metarray->id];
9834         m = &metarray_runtime->metarray[metarray_index];
9835         mp_old = m->profile;
9836
9837         meter_init(m);
9838
9839         mp_old->n_users--;
9840
9841         return 0;
9842 }
9843
9844 int
9845 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9846                       const char *metarray_name,
9847                       uint32_t metarray_index,
9848                       const char *profile_name)
9849 {
9850         struct meter_profile *mp, *mp_old;
9851         struct metarray *metarray;
9852         struct metarray_runtime *metarray_runtime;
9853         struct meter *m;
9854
9855         CHECK(p, EINVAL);
9856         CHECK_NAME(metarray_name, EINVAL);
9857
9858         metarray = metarray_find(p, metarray_name);
9859         CHECK(metarray, EINVAL);
9860         CHECK(metarray_index < metarray->size, EINVAL);
9861
9862         mp = meter_profile_find(p, profile_name);
9863         CHECK(mp, EINVAL);
9864
9865         metarray_runtime = &p->metarray_runtime[metarray->id];
9866         m = &metarray_runtime->metarray[metarray_index];
9867         mp_old = m->profile;
9868
9869         memset(m, 0, sizeof(struct meter));
9870         rte_meter_trtcm_config(&m->m, &mp->profile);
9871         m->profile = mp;
9872         m->color_mask = RTE_COLORS;
9873
9874         mp->n_users++;
9875         mp_old->n_users--;
9876
9877         return 0;
9878 }
9879
9880 int
9881 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9882                              const char *metarray_name,
9883                              uint32_t metarray_index,
9884                              struct rte_swx_ctl_meter_stats *stats)
9885 {
9886         struct metarray *metarray;
9887         struct metarray_runtime *metarray_runtime;
9888         struct meter *m;
9889
9890         CHECK(p, EINVAL);
9891         CHECK_NAME(metarray_name, EINVAL);
9892
9893         metarray = metarray_find(p, metarray_name);
9894         CHECK(metarray, EINVAL);
9895         CHECK(metarray_index < metarray->size, EINVAL);
9896
9897         CHECK(stats, EINVAL);
9898
9899         metarray_runtime = &p->metarray_runtime[metarray->id];
9900         m = &metarray_runtime->metarray[metarray_index];
9901
9902         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9903         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9904
9905         return 0;
9906 }
9907
9908 /*
9909  * Pipeline compilation.
9910  */
9911 static const char *
9912 instr_type_to_name(struct instruction *instr)
9913 {
9914         switch (instr->type) {
9915         case INSTR_RX: return "INSTR_RX";
9916
9917         case INSTR_TX: return "INSTR_TX";
9918         case INSTR_TX_I: return "INSTR_TX_I";
9919         case INSTR_DROP: return "INSTR_DROP";
9920
9921         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
9922         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
9923         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
9924         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
9925         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
9926         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
9927         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
9928         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
9929
9930         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
9931
9932         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
9933
9934         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
9935         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
9936         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
9937         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
9938         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
9939         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
9940         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
9941         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
9942         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
9943
9944         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
9945         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
9946
9947         case INSTR_MOV: return "INSTR_MOV";
9948         case INSTR_MOV_MH: return "INSTR_MOV_MH";
9949         case INSTR_MOV_HM: return "INSTR_MOV_HM";
9950         case INSTR_MOV_HH: return "INSTR_MOV_HH";
9951         case INSTR_MOV_I: return "INSTR_MOV_I";
9952
9953         case INSTR_DMA_HT: return "INSTR_DMA_HT";
9954         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
9955         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
9956         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
9957         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
9958         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
9959         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
9960         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
9961
9962         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
9963         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
9964         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
9965         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
9966         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
9967         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
9968
9969         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
9970         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
9971         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
9972         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
9973         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
9974         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
9975
9976         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
9977         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
9978         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
9979         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
9980
9981         case INSTR_ALU_AND: return "INSTR_ALU_AND";
9982         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
9983         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
9984         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
9985         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
9986
9987         case INSTR_ALU_OR: return "INSTR_ALU_OR";
9988         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
9989         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
9990         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
9991         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
9992
9993         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
9994         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
9995         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
9996         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
9997         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
9998
9999         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10000         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10001         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10002         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10003         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10004         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10005
10006         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10007         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10008         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10009         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10010         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10011         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10012
10013         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10014         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10015         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10016
10017         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10018         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10019         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10020         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10021         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10022         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10023
10024         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10025         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10026         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10027         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10028         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10029         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10030         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10031         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10032         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10033
10034         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10035         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10036         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10037         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10038         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10039         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10040         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10041         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10042         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10043
10044         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10045         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10046         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10047
10048         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10049         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10050         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10051         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10052         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10053         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10054         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10055         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10056         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10057         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10058         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10059         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10060
10061         case INSTR_TABLE: return "INSTR_TABLE";
10062         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10063         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10064         case INSTR_LEARNER: return "INSTR_LEARNER";
10065         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10066
10067         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10068         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10069
10070         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10071         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10072
10073         case INSTR_JMP: return "INSTR_JMP";
10074         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10075         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10076         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10077         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10078         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10079         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10080         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10081         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10082         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10083         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10084         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10085         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10086         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10087         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10088         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10089         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10090         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10091         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10092         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10093         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10094         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10095         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10096         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10097         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10098         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10099         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10100         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10101         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10102
10103         case INSTR_RETURN: return "INSTR_RETURN";
10104
10105         default: return "INSTR_UNKNOWN";
10106         }
10107 }
10108
10109 typedef void
10110 (*instruction_export_t)(struct instruction *, FILE *);
10111
10112 static void
10113 instr_io_export(struct instruction *instr, FILE *f)
10114 {
10115         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10116
10117         /* n_io, n_io_imm, n_hdrs. */
10118         if (instr->type == INSTR_RX ||
10119             instr->type == INSTR_TX ||
10120             instr->type == INSTR_HDR_EXTRACT_M ||
10121             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10122                 n_io = 1;
10123
10124         if (instr->type == INSTR_TX_I)
10125                 n_io_imm = 1;
10126
10127         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10128                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10129
10130         if (instr->type == INSTR_HDR_EXTRACT_M ||
10131             instr->type == INSTR_HDR_LOOKAHEAD ||
10132             instr->type == INSTR_HDR_EMIT)
10133                 n_hdrs = 1;
10134
10135         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10136                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10137
10138         /* instr. */
10139         fprintf(f,
10140                 "\t{\n"
10141                 "\t\t.type = %s,\n",
10142                 instr_type_to_name(instr));
10143
10144         /* instr.io. */
10145         if (n_io || n_io_imm || n_hdrs)
10146                 fprintf(f,
10147                         "\t\t.io = {\n");
10148
10149         /* instr.io.io. */
10150         if (n_io)
10151                 fprintf(f,
10152                         "\t\t\t.io = {\n"
10153                         "\t\t\t\t.offset = %u,\n"
10154                         "\t\t\t\t.n_bits = %u,\n"
10155                         "\t\t\t},\n",
10156                         instr->io.io.offset,
10157                         instr->io.io.n_bits);
10158
10159         if (n_io_imm)
10160                 fprintf(f,
10161                         "\t\t\t.io = {\n"
10162                         "\t\t\t\t.val = %u,\n"
10163                         "\t\t\t},\n",
10164                         instr->io.io.val);
10165
10166         /* instr.io.hdr. */
10167         if (n_hdrs) {
10168                 fprintf(f,
10169                         "\t\t.hdr = {\n");
10170
10171                 /* instr.io.hdr.header_id. */
10172                 fprintf(f,
10173                         "\t\t\t.header_id = {");
10174
10175                 for (i = 0; i < n_hdrs; i++)
10176                         fprintf(f,
10177                                 "%u, ",
10178                                 instr->io.hdr.header_id[i]);
10179
10180                 fprintf(f,
10181                         "},\n");
10182
10183                 /* instr.io.hdr.struct_id. */
10184                 fprintf(f,
10185                         "\t\t\t.struct_id = {");
10186
10187                 for (i = 0; i < n_hdrs; i++)
10188                         fprintf(f,
10189                                 "%u, ",
10190                                 instr->io.hdr.struct_id[i]);
10191
10192                 fprintf(f,
10193                         "},\n");
10194
10195                 /* instr.io.hdr.n_bytes. */
10196                 fprintf(f,
10197                         "\t\t\t.n_bytes = {");
10198
10199                 for (i = 0; i < n_hdrs; i++)
10200                         fprintf(f,
10201                                 "%u, ",
10202                                 instr->io.hdr.n_bytes[i]);
10203
10204                 fprintf(f,
10205                         "},\n");
10206
10207                 /* instr.io.hdr - closing curly brace. */
10208                 fprintf(f,
10209                         "\t\t\t}\n,");
10210         }
10211
10212         /* instr.io - closing curly brace. */
10213         if (n_io || n_io_imm || n_hdrs)
10214                 fprintf(f,
10215                         "\t\t},\n");
10216
10217         /* instr - closing curly brace. */
10218         fprintf(f,
10219                 "\t},\n");
10220 }
10221
10222 static void
10223 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10224 {
10225         fprintf(f,
10226                 "\t{\n"
10227                 "\t\t.type = %s,\n"
10228                 "\t\t.valid = {\n"
10229                 "\t\t\t.header_id = %u,\n"
10230                 "\t\t},\n"
10231                 "\t},\n",
10232                 instr_type_to_name(instr),
10233                 instr->valid.header_id);
10234 }
10235
10236 static void
10237 instr_mov_export(struct instruction *instr, FILE *f)
10238 {
10239         if (instr->type != INSTR_MOV_I)
10240                 fprintf(f,
10241                         "\t{\n"
10242                         "\t\t.type = %s,\n"
10243                         "\t\t.mov = {\n"
10244                         "\t\t\t.dst = {\n"
10245                         "\t\t\t\t.struct_id = %u,\n"
10246                         "\t\t\t\t.n_bits = %u,\n"
10247                         "\t\t\t\t.offset = %u,\n"
10248                         "\t\t\t},\n"
10249                         "\t\t\t.src = {\n"
10250                         "\t\t\t\t.struct_id = %u,\n"
10251                         "\t\t\t\t.n_bits = %u,\n"
10252                         "\t\t\t\t.offset = %u,\n"
10253                         "\t\t\t},\n"
10254                         "\t\t},\n"
10255                         "\t},\n",
10256                         instr_type_to_name(instr),
10257                         instr->mov.dst.struct_id,
10258                         instr->mov.dst.n_bits,
10259                         instr->mov.dst.offset,
10260                         instr->mov.src.struct_id,
10261                         instr->mov.src.n_bits,
10262                         instr->mov.src.offset);
10263         else
10264                 fprintf(f,
10265                         "\t{\n"
10266                         "\t\t.type = %s,\n"
10267                         "\t\t.mov = {\n"
10268                         "\t\t\t.dst = {\n"
10269                         "\t\t\t\t.struct_id = %u,\n"
10270                         "\t\t\t\t.n_bits = %u,\n"
10271                         "\t\t\t\t.offset = %u,\n"
10272                         "\t\t\t}\n,"
10273                         "\t\t\t.src_val = %" PRIu64 ",\n"
10274                         "\t\t},\n"
10275                         "\t},\n",
10276                         instr_type_to_name(instr),
10277                         instr->mov.dst.struct_id,
10278                         instr->mov.dst.n_bits,
10279                         instr->mov.dst.offset,
10280                         instr->mov.src_val);
10281 }
10282
10283 static void
10284 instr_dma_ht_export(struct instruction *instr, FILE *f)
10285 {
10286         uint32_t n_dma = 0, i;
10287
10288         /* n_dma. */
10289         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10290
10291         /* instr. */
10292         fprintf(f,
10293                 "\t{\n"
10294                 "\t\t.type = %s,\n",
10295                 instr_type_to_name(instr));
10296
10297         /* instr.dma. */
10298         fprintf(f,
10299                 "\t\t.dma = {\n");
10300
10301         /* instr.dma.dst. */
10302         fprintf(f,
10303                 "\t\t\t.dst = {\n");
10304
10305         /* instr.dma.dst.header_id. */
10306         fprintf(f,
10307                 "\t\t\t\t.header_id = {");
10308
10309         for (i = 0; i < n_dma; i++)
10310                 fprintf(f,
10311                         "%u, ",
10312                         instr->dma.dst.header_id[i]);
10313
10314         fprintf(f,
10315                 "},\n");
10316
10317         /* instr.dma.dst.struct_id. */
10318         fprintf(f,
10319                 "\t\t\t\t.struct_id = {");
10320
10321         for (i = 0; i < n_dma; i++)
10322                 fprintf(f,
10323                         "%u, ",
10324                         instr->dma.dst.struct_id[i]);
10325
10326         fprintf(f,
10327                 "},\n");
10328
10329         /* instr.dma.dst - closing curly brace. */
10330         fprintf(f,
10331                 "\t\t\t},\n");
10332
10333         /* instr.dma.src. */
10334         fprintf(f,
10335                 "\t\t\t.src = {\n");
10336
10337         /* instr.dma.src.offset. */
10338         fprintf(f,
10339                 "\t\t\t\t.offset = {");
10340
10341         for (i = 0; i < n_dma; i++)
10342                 fprintf(f,
10343                         "%u, ",
10344                         instr->dma.src.offset[i]);
10345
10346         fprintf(f,
10347                 "},\n");
10348
10349         /* instr.dma.src - closing curly brace. */
10350         fprintf(f,
10351                 "\t\t\t},\n");
10352
10353         /* instr.dma.n_bytes. */
10354         fprintf(f,
10355                 "\t\t\t.n_bytes = {");
10356
10357         for (i = 0; i < n_dma; i++)
10358                 fprintf(f,
10359                         "%u, ",
10360                         instr->dma.n_bytes[i]);
10361
10362         fprintf(f,
10363                 "},\n");
10364
10365         /* instr.dma - closing curly brace. */
10366         fprintf(f,
10367                 "\t\t},\n");
10368
10369         /* instr - closing curly brace. */
10370         fprintf(f,
10371                 "\t},\n");
10372 }
10373
10374 static void
10375 instr_alu_export(struct instruction *instr, FILE *f)
10376 {
10377         int imm = 0;
10378
10379         if (instr->type == INSTR_ALU_ADD_MI ||
10380             instr->type == INSTR_ALU_ADD_HI ||
10381             instr->type == INSTR_ALU_SUB_MI ||
10382             instr->type == INSTR_ALU_SUB_HI ||
10383             instr->type == INSTR_ALU_SHL_MI ||
10384             instr->type == INSTR_ALU_SHL_HI ||
10385             instr->type == INSTR_ALU_SHR_MI ||
10386             instr->type == INSTR_ALU_SHR_HI ||
10387             instr->type == INSTR_ALU_AND_I ||
10388             instr->type == INSTR_ALU_OR_I ||
10389             instr->type == INSTR_ALU_XOR_I)
10390                 imm = 1;
10391
10392         if (!imm)
10393                 fprintf(f,
10394                         "\t{\n"
10395                         "\t\t.type = %s,\n"
10396                         "\t\t.alu = {\n"
10397                         "\t\t\t.dst = {\n"
10398                         "\t\t\t\t.struct_id = %u,\n"
10399                         "\t\t\t\t.n_bits = %u,\n"
10400                         "\t\t\t\t.offset = %u,\n"
10401                         "\t\t\t},\n"
10402                         "\t\t\t.src = {\n"
10403                         "\t\t\t\t.struct_id = %u,\n"
10404                         "\t\t\t\t.n_bits = %u,\n"
10405                         "\t\t\t\t.offset = %u,\n"
10406                         "\t\t\t},\n"
10407                         "\t\t},\n"
10408                         "\t},\n",
10409                         instr_type_to_name(instr),
10410                         instr->alu.dst.struct_id,
10411                         instr->alu.dst.n_bits,
10412                         instr->alu.dst.offset,
10413                         instr->alu.src.struct_id,
10414                         instr->alu.src.n_bits,
10415                         instr->alu.src.offset);
10416         else
10417                 fprintf(f,
10418                         "\t{\n"
10419                         "\t\t.type = %s,\n"
10420                         "\t\t.alu = {\n"
10421                         "\t\t\t.dst = {\n"
10422                         "\t\t\t\t.struct_id = %u,\n"
10423                         "\t\t\t\t.n_bits = %u,\n"
10424                         "\t\t\t\t.offset = %u,\n"
10425                         "\t\t\t}\n,"
10426                         "\t\t\t.src_val = %" PRIu64 ",\n"
10427                         "\t\t},\n"
10428                         "\t},\n",
10429                         instr_type_to_name(instr),
10430                         instr->alu.dst.struct_id,
10431                         instr->alu.dst.n_bits,
10432                         instr->alu.dst.offset,
10433                         instr->alu.src_val);
10434 }
10435
10436 static void
10437 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10438 {
10439         int prefetch  = 0, idx_imm = 0, src_imm = 0;
10440
10441         if (instr->type == INSTR_REGPREFETCH_RH ||
10442             instr->type == INSTR_REGPREFETCH_RM ||
10443             instr->type == INSTR_REGPREFETCH_RI)
10444                 prefetch = 1;
10445
10446         /* index is the 3rd operand for the regrd instruction and the 2nd
10447          * operand for the regwr and regadd instructions.
10448          */
10449         if (instr->type == INSTR_REGPREFETCH_RI ||
10450             instr->type == INSTR_REGRD_HRI ||
10451             instr->type == INSTR_REGRD_MRI ||
10452             instr->type == INSTR_REGWR_RIH ||
10453             instr->type == INSTR_REGWR_RIM ||
10454             instr->type == INSTR_REGWR_RII ||
10455             instr->type == INSTR_REGADD_RIH ||
10456             instr->type == INSTR_REGADD_RIM ||
10457             instr->type == INSTR_REGADD_RII)
10458                 idx_imm = 1;
10459
10460         /* src is the 3rd operand for the regwr and regadd instructions. */
10461         if (instr->type == INSTR_REGWR_RHI ||
10462             instr->type == INSTR_REGWR_RMI ||
10463             instr->type == INSTR_REGWR_RII ||
10464             instr->type == INSTR_REGADD_RHI ||
10465             instr->type == INSTR_REGADD_RMI ||
10466             instr->type == INSTR_REGADD_RII)
10467                 src_imm = 1;
10468
10469         /* instr.regarray.regarray_id. */
10470         fprintf(f,
10471                 "\t{\n"
10472                 "\t\t.type = %s,\n"
10473                 "\t\t.regarray = {\n"
10474                 "\t\t\t.regarray_id = %u,\n",
10475                 instr_type_to_name(instr),
10476                 instr->regarray.regarray_id);
10477
10478         /* instr.regarray.idx / instr.regarray.idx_val. */
10479         if (!idx_imm)
10480                 fprintf(f,
10481                         "\t\t\t\t.idx = {\n"
10482                         "\t\t\t\t\t.struct_id = %u,\n"
10483                         "\t\t\t\t\t.n_bits = %u,\n"
10484                         "\t\t\t\t\t.offset = %u,\n"
10485                         "\t\t\t\t},\n",
10486                         instr->regarray.idx.struct_id,
10487                         instr->regarray.idx.n_bits,
10488                         instr->regarray.idx.offset);
10489         else
10490                 fprintf(f,
10491                         "\t\t\t\t.idx_val = %u,\n",
10492                         instr->regarray.idx_val);
10493
10494         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10495         if (!prefetch) {
10496                 if (!src_imm)
10497                         fprintf(f,
10498                                 "\t\t\t\t.dstsrc = {\n"
10499                                 "\t\t\t\t\t.struct_id = %u,\n"
10500                                 "\t\t\t\t\t.n_bits = %u,\n"
10501                                 "\t\t\t\t\t.offset = %u,\n"
10502                                 "\t\t\t\t},\n",
10503                                 instr->regarray.dstsrc.struct_id,
10504                                 instr->regarray.dstsrc.n_bits,
10505                                 instr->regarray.dstsrc.offset);
10506                 else
10507                         fprintf(f,
10508                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10509                                 instr->regarray.dstsrc_val);
10510         }
10511
10512         /* instr.regarray and instr - closing curly braces. */
10513         fprintf(f,
10514                 "\t\t},\n"
10515                 "\t},\n");
10516 }
10517
10518 static void
10519 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10520 {
10521         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
10522
10523         if (instr->type == INSTR_METPREFETCH_H ||
10524             instr->type == INSTR_METPREFETCH_M ||
10525             instr->type == INSTR_METPREFETCH_I)
10526                 prefetch = 1;
10527
10528         /* idx_imm. */
10529         if (instr->type == INSTR_METPREFETCH_I ||
10530             instr->type == INSTR_METER_IHM ||
10531             instr->type == INSTR_METER_IHI ||
10532             instr->type == INSTR_METER_IMM ||
10533             instr->type == INSTR_METER_IMI)
10534                 idx_imm = 1;
10535
10536         /* color_in_imm. */
10537         if (instr->type == INSTR_METER_HHI ||
10538             instr->type == INSTR_METER_HMI ||
10539             instr->type == INSTR_METER_MHI ||
10540             instr->type == INSTR_METER_MMI ||
10541             instr->type == INSTR_METER_IHI ||
10542             instr->type == INSTR_METER_IMI)
10543                 color_in_imm = 1;
10544
10545         /* instr.meter.metarray_id. */
10546         fprintf(f,
10547                 "\t{\n"
10548                 "\t\t.type = %s,\n"
10549                 "\t\t.meter = {\n"
10550                 "\t\t\t.metarray_id = %u,\n",
10551                 instr_type_to_name(instr),
10552                 instr->meter.metarray_id);
10553
10554         /* instr.meter.idx / instr.meter.idx_val. */
10555         if (!idx_imm)
10556                 fprintf(f,
10557                         "\t\t\t.idx = {\n"
10558                         "\t\t\t\t.struct_id = %u,\n"
10559                         "\t\t\t\t.n_bits = %u,\n"
10560                         "\t\t\t\t.offset = %u,\n"
10561                         "\t\t\t},\n",
10562                         instr->meter.idx.struct_id,
10563                         instr->meter.idx.n_bits,
10564                         instr->meter.idx.offset);
10565         else
10566                 fprintf(f,
10567                         "\t\t\t.idx_val = %u,\n",
10568                         instr->meter.idx_val);
10569
10570         if (!prefetch) {
10571                 /* instr.meter.length. */
10572                 fprintf(f,
10573                         "\t\t\t.length = {\n"
10574                         "\t\t\t\t.struct_id = %u,\n"
10575                         "\t\t\t\t.n_bits = %u,\n"
10576                         "\t\t\t\t.offset = %u,\n"
10577                         "\t\t\t},\n",
10578                         instr->meter.length.struct_id,
10579                         instr->meter.length.n_bits,
10580                         instr->meter.length.offset);
10581
10582                 /* instr.meter.color_in / instr.meter.color_in_val. */
10583                 if (!color_in_imm)
10584                         fprintf(f,
10585                                 "\t\t\t.color_in = {\n"
10586                                 "\t\t\t\t.struct_id = %u,\n"
10587                                 "\t\t\t\t.n_bits = %u,\n"
10588                                 "\t\t\t\t.offset = %u,\n"
10589                                 "\t\t\t},\n",
10590                                 instr->meter.color_in.struct_id,
10591                                 instr->meter.color_in.n_bits,
10592                                 instr->meter.color_in.offset);
10593                 else
10594                         fprintf(f,
10595                                 "\t\t\t.color_in_val = %u,\n",
10596                                 (uint32_t)instr->meter.color_in_val);
10597
10598                 /* instr.meter.color_out. */
10599                 fprintf(f,
10600                         "\t\t\t.color_out = {\n"
10601                         "\t\t\t\t.struct_id = %u,\n"
10602                         "\t\t\t\t.n_bits = %u,\n"
10603                         "\t\t\t\t.offset = %u,\n"
10604                         "\t\t\t},\n",
10605                         instr->meter.color_out.struct_id,
10606                         instr->meter.color_out.n_bits,
10607                         instr->meter.color_out.offset);
10608         }
10609
10610         /* instr.meter and instr - closing curly braces. */
10611         fprintf(f,
10612                 "\t\t},\n"
10613                 "\t},\n");
10614 }
10615
10616 static void
10617 instr_table_export(struct instruction *instr,
10618                 FILE *f)
10619 {
10620         fprintf(f,
10621                 "\t{\n"
10622                 "\t\t.type = %s,\n"
10623                 "\t\t.table = {\n"
10624                 "\t\t\t.table_id = %u,\n"
10625                 "\t\t},\n"
10626                 "\t},\n",
10627                 instr_type_to_name(instr),
10628                 instr->table.table_id);
10629 }
10630
10631 static void
10632 instr_learn_export(struct instruction *instr, FILE *f)
10633 {
10634         fprintf(f,
10635                 "\t{\n"
10636                 "\t\t.type = %s,\n"
10637                 "\t\t.learn = {\n"
10638                 "\t\t\t\t.action_id = %u,\n"
10639                 "\t\t},\n"
10640                 "\t},\n",
10641                 instr_type_to_name(instr),
10642                 instr->learn.action_id);
10643 }
10644
10645 static void
10646 instr_forget_export(struct instruction *instr, FILE *f)
10647 {
10648         fprintf(f,
10649                 "\t{\n"
10650                 "\t\t.type = %s,\n"
10651                 "\t},\n",
10652                 instr_type_to_name(instr));
10653 }
10654
10655 static void
10656 instr_extern_export(struct instruction *instr, FILE *f)
10657 {
10658         if (instr->type == INSTR_EXTERN_OBJ)
10659                 fprintf(f,
10660                         "\t{\n"
10661                         "\t\t.type = %s,\n"
10662                         "\t\t.ext_obj = {\n"
10663                         "\t\t\t.ext_obj_id = %u,\n"
10664                         "\t\t\t.func_id = %u,\n"
10665                         "\t\t},\n"
10666                         "\t},\n",
10667                         instr_type_to_name(instr),
10668                         instr->ext_obj.ext_obj_id,
10669                         instr->ext_obj.func_id);
10670         else
10671                 fprintf(f,
10672                         "\t{\n"
10673                         "\t\t.type = %s,\n"
10674                         "\t\t.ext_func = {\n"
10675                         "\t\t\t.ext_func_id = %u,\n"
10676                         "\t\t},\n"
10677                         "\t},\n",
10678                         instr_type_to_name(instr),
10679                         instr->ext_func.ext_func_id);
10680 }
10681
10682 static void
10683 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10684 {
10685         fprintf(f,
10686                 "\t{\n"
10687                 "\t\t.type = %s,\n"
10688                 "\t\t.jmp = {\n"
10689                 "\t\t\t.ip = NULL,\n",
10690                 instr_type_to_name(instr));
10691
10692         switch (instr->type) {
10693         case INSTR_JMP_VALID:
10694         case INSTR_JMP_INVALID:
10695                 fprintf(f,
10696                         "\t\t\t.header_id = %u,\n",
10697                         instr->jmp.header_id);
10698                 break;
10699
10700         case INSTR_JMP_ACTION_HIT:
10701         case INSTR_JMP_ACTION_MISS:
10702                 fprintf(f,
10703                         "\t\t\t.action_id = %u,\n",
10704                         instr->jmp.action_id);
10705                 break;
10706
10707         case INSTR_JMP_EQ:
10708         case INSTR_JMP_EQ_MH:
10709         case INSTR_JMP_EQ_HM:
10710         case INSTR_JMP_EQ_HH:
10711         case INSTR_JMP_NEQ:
10712         case INSTR_JMP_NEQ_MH:
10713         case INSTR_JMP_NEQ_HM:
10714         case INSTR_JMP_NEQ_HH:
10715         case INSTR_JMP_LT:
10716         case INSTR_JMP_LT_MH:
10717         case INSTR_JMP_LT_HM:
10718         case INSTR_JMP_LT_HH:
10719         case INSTR_JMP_GT:
10720         case INSTR_JMP_GT_MH:
10721         case INSTR_JMP_GT_HM:
10722         case INSTR_JMP_GT_HH:
10723                 fprintf(f,
10724                         "\t\t\t.a = {\n"
10725                         "\t\t\t\t.struct_id = %u,\n"
10726                         "\t\t\t\t.n_bits = %u,\n"
10727                         "\t\t\t\t.offset = %u,\n"
10728                         "\t\t\t},\n"
10729                         "\t\t\t.b = {\n"
10730                         "\t\t\t\t.struct_id = %u,\n"
10731                         "\t\t\t\t.n_bits = %u,\n"
10732                         "\t\t\t\t.offset = %u,\n"
10733                         "\t\t\t},\n",
10734                         instr->jmp.a.struct_id,
10735                         instr->jmp.a.n_bits,
10736                         instr->jmp.a.offset,
10737                         instr->jmp.b.struct_id,
10738                         instr->jmp.b.n_bits,
10739                         instr->jmp.b.offset);
10740                 break;
10741
10742         case INSTR_JMP_EQ_I:
10743         case INSTR_JMP_NEQ_I:
10744         case INSTR_JMP_LT_MI:
10745         case INSTR_JMP_LT_HI:
10746         case INSTR_JMP_GT_MI:
10747         case INSTR_JMP_GT_HI:
10748                 fprintf(f,
10749                         "\t\t\t.a = {\n"
10750                         "\t\t\t\t.struct_id = %u,\n"
10751                         "\t\t\t\t.n_bits = %u,\n"
10752                         "\t\t\t\t.offset = %u,\n"
10753                         "\t\t\t}\n,"
10754                         "\t\t\t.b_val = %" PRIu64 ",\n",
10755                         instr->jmp.a.struct_id,
10756                         instr->jmp.a.n_bits,
10757                         instr->jmp.a.offset,
10758                         instr->jmp.b_val);
10759                 break;
10760
10761         default:
10762                 break;
10763         }
10764
10765         fprintf(f,
10766                 "\t\t},\n"
10767                 "\t},\n");
10768 }
10769
10770 static void
10771 instr_return_export(struct instruction *instr,
10772                 FILE *f)
10773 {
10774         fprintf(f,
10775                 "\t{\n"
10776                 "\t\t.type = %s,\n",
10777                 instr_type_to_name(instr));
10778
10779         fprintf(f,
10780                 "\t},\n");
10781 }
10782
10783 static instruction_export_t export_table[] = {
10784         [INSTR_RX] = instr_io_export,
10785
10786         [INSTR_TX] = instr_io_export,
10787         [INSTR_TX_I] = instr_io_export,
10788         [INSTR_DROP] = instr_io_export,
10789
10790         [INSTR_HDR_EXTRACT] = instr_io_export,
10791         [INSTR_HDR_EXTRACT2] = instr_io_export,
10792         [INSTR_HDR_EXTRACT3] = instr_io_export,
10793         [INSTR_HDR_EXTRACT4] = instr_io_export,
10794         [INSTR_HDR_EXTRACT5] = instr_io_export,
10795         [INSTR_HDR_EXTRACT6] = instr_io_export,
10796         [INSTR_HDR_EXTRACT7] = instr_io_export,
10797         [INSTR_HDR_EXTRACT8] = instr_io_export,
10798
10799         [INSTR_HDR_EXTRACT_M] = instr_io_export,
10800
10801         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10802
10803         [INSTR_HDR_EMIT] = instr_io_export,
10804         [INSTR_HDR_EMIT_TX] = instr_io_export,
10805         [INSTR_HDR_EMIT2_TX] = instr_io_export,
10806         [INSTR_HDR_EMIT3_TX] = instr_io_export,
10807         [INSTR_HDR_EMIT4_TX] = instr_io_export,
10808         [INSTR_HDR_EMIT5_TX] = instr_io_export,
10809         [INSTR_HDR_EMIT6_TX] = instr_io_export,
10810         [INSTR_HDR_EMIT7_TX] = instr_io_export,
10811         [INSTR_HDR_EMIT8_TX] = instr_io_export,
10812
10813         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10814         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10815
10816         [INSTR_MOV] = instr_mov_export,
10817         [INSTR_MOV_MH] = instr_mov_export,
10818         [INSTR_MOV_HM] = instr_mov_export,
10819         [INSTR_MOV_HH] = instr_mov_export,
10820         [INSTR_MOV_I] = instr_mov_export,
10821
10822         [INSTR_DMA_HT]  = instr_dma_ht_export,
10823         [INSTR_DMA_HT2] = instr_dma_ht_export,
10824         [INSTR_DMA_HT3] = instr_dma_ht_export,
10825         [INSTR_DMA_HT4] = instr_dma_ht_export,
10826         [INSTR_DMA_HT5] = instr_dma_ht_export,
10827         [INSTR_DMA_HT6] = instr_dma_ht_export,
10828         [INSTR_DMA_HT7] = instr_dma_ht_export,
10829         [INSTR_DMA_HT8] = instr_dma_ht_export,
10830
10831         [INSTR_ALU_ADD] = instr_alu_export,
10832         [INSTR_ALU_ADD_MH] = instr_alu_export,
10833         [INSTR_ALU_ADD_HM] = instr_alu_export,
10834         [INSTR_ALU_ADD_HH] = instr_alu_export,
10835         [INSTR_ALU_ADD_MI] = instr_alu_export,
10836         [INSTR_ALU_ADD_HI] = instr_alu_export,
10837
10838         [INSTR_ALU_SUB] = instr_alu_export,
10839         [INSTR_ALU_SUB_MH] = instr_alu_export,
10840         [INSTR_ALU_SUB_HM] = instr_alu_export,
10841         [INSTR_ALU_SUB_HH] = instr_alu_export,
10842         [INSTR_ALU_SUB_MI] = instr_alu_export,
10843         [INSTR_ALU_SUB_HI] = instr_alu_export,
10844
10845         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10846         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10847         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10848         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10849
10850         [INSTR_ALU_AND] = instr_alu_export,
10851         [INSTR_ALU_AND_MH] = instr_alu_export,
10852         [INSTR_ALU_AND_HM] = instr_alu_export,
10853         [INSTR_ALU_AND_HH] = instr_alu_export,
10854         [INSTR_ALU_AND_I] = instr_alu_export,
10855
10856         [INSTR_ALU_OR] = instr_alu_export,
10857         [INSTR_ALU_OR_MH] = instr_alu_export,
10858         [INSTR_ALU_OR_HM] = instr_alu_export,
10859         [INSTR_ALU_OR_HH] = instr_alu_export,
10860         [INSTR_ALU_OR_I] = instr_alu_export,
10861
10862         [INSTR_ALU_XOR] = instr_alu_export,
10863         [INSTR_ALU_XOR_MH] = instr_alu_export,
10864         [INSTR_ALU_XOR_HM] = instr_alu_export,
10865         [INSTR_ALU_XOR_HH] = instr_alu_export,
10866         [INSTR_ALU_XOR_I] = instr_alu_export,
10867
10868         [INSTR_ALU_SHL] = instr_alu_export,
10869         [INSTR_ALU_SHL_MH] = instr_alu_export,
10870         [INSTR_ALU_SHL_HM] = instr_alu_export,
10871         [INSTR_ALU_SHL_HH] = instr_alu_export,
10872         [INSTR_ALU_SHL_MI] = instr_alu_export,
10873         [INSTR_ALU_SHL_HI] = instr_alu_export,
10874
10875         [INSTR_ALU_SHR] = instr_alu_export,
10876         [INSTR_ALU_SHR_MH] = instr_alu_export,
10877         [INSTR_ALU_SHR_HM] = instr_alu_export,
10878         [INSTR_ALU_SHR_HH] = instr_alu_export,
10879         [INSTR_ALU_SHR_MI] = instr_alu_export,
10880         [INSTR_ALU_SHR_HI] = instr_alu_export,
10881
10882         [INSTR_REGPREFETCH_RH] = instr_reg_export,
10883         [INSTR_REGPREFETCH_RM] = instr_reg_export,
10884         [INSTR_REGPREFETCH_RI] = instr_reg_export,
10885
10886         [INSTR_REGRD_HRH] = instr_reg_export,
10887         [INSTR_REGRD_HRM] = instr_reg_export,
10888         [INSTR_REGRD_MRH] = instr_reg_export,
10889         [INSTR_REGRD_MRM] = instr_reg_export,
10890         [INSTR_REGRD_HRI] = instr_reg_export,
10891         [INSTR_REGRD_MRI] = instr_reg_export,
10892
10893         [INSTR_REGWR_RHH] = instr_reg_export,
10894         [INSTR_REGWR_RHM] = instr_reg_export,
10895         [INSTR_REGWR_RMH] = instr_reg_export,
10896         [INSTR_REGWR_RMM] = instr_reg_export,
10897         [INSTR_REGWR_RHI] = instr_reg_export,
10898         [INSTR_REGWR_RMI] = instr_reg_export,
10899         [INSTR_REGWR_RIH] = instr_reg_export,
10900         [INSTR_REGWR_RIM] = instr_reg_export,
10901         [INSTR_REGWR_RII] = instr_reg_export,
10902
10903         [INSTR_REGADD_RHH] = instr_reg_export,
10904         [INSTR_REGADD_RHM] = instr_reg_export,
10905         [INSTR_REGADD_RMH] = instr_reg_export,
10906         [INSTR_REGADD_RMM] = instr_reg_export,
10907         [INSTR_REGADD_RHI] = instr_reg_export,
10908         [INSTR_REGADD_RMI] = instr_reg_export,
10909         [INSTR_REGADD_RIH] = instr_reg_export,
10910         [INSTR_REGADD_RIM] = instr_reg_export,
10911         [INSTR_REGADD_RII] = instr_reg_export,
10912
10913         [INSTR_METPREFETCH_H] = instr_meter_export,
10914         [INSTR_METPREFETCH_M] = instr_meter_export,
10915         [INSTR_METPREFETCH_I] = instr_meter_export,
10916
10917         [INSTR_METER_HHM] = instr_meter_export,
10918         [INSTR_METER_HHI] = instr_meter_export,
10919         [INSTR_METER_HMM] = instr_meter_export,
10920         [INSTR_METER_HMI] = instr_meter_export,
10921         [INSTR_METER_MHM] = instr_meter_export,
10922         [INSTR_METER_MHI] = instr_meter_export,
10923         [INSTR_METER_MMM] = instr_meter_export,
10924         [INSTR_METER_MMI] = instr_meter_export,
10925         [INSTR_METER_IHM] = instr_meter_export,
10926         [INSTR_METER_IHI] = instr_meter_export,
10927         [INSTR_METER_IMM] = instr_meter_export,
10928         [INSTR_METER_IMI] = instr_meter_export,
10929
10930         [INSTR_TABLE] = instr_table_export,
10931         [INSTR_TABLE_AF] = instr_table_export,
10932         [INSTR_SELECTOR] = instr_table_export,
10933         [INSTR_LEARNER] = instr_table_export,
10934         [INSTR_LEARNER_AF] = instr_table_export,
10935
10936         [INSTR_LEARNER_LEARN] = instr_learn_export,
10937         [INSTR_LEARNER_FORGET] = instr_forget_export,
10938
10939         [INSTR_EXTERN_OBJ] = instr_extern_export,
10940         [INSTR_EXTERN_FUNC] = instr_extern_export,
10941
10942         [INSTR_JMP] = instr_jmp_export,
10943         [INSTR_JMP_VALID] = instr_jmp_export,
10944         [INSTR_JMP_INVALID] = instr_jmp_export,
10945         [INSTR_JMP_HIT] = instr_jmp_export,
10946         [INSTR_JMP_MISS] = instr_jmp_export,
10947         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
10948         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
10949
10950         [INSTR_JMP_EQ] = instr_jmp_export,
10951         [INSTR_JMP_EQ_MH] = instr_jmp_export,
10952         [INSTR_JMP_EQ_HM] = instr_jmp_export,
10953         [INSTR_JMP_EQ_HH] = instr_jmp_export,
10954         [INSTR_JMP_EQ_I] = instr_jmp_export,
10955
10956         [INSTR_JMP_NEQ] = instr_jmp_export,
10957         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
10958         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
10959         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
10960         [INSTR_JMP_NEQ_I] = instr_jmp_export,
10961
10962         [INSTR_JMP_LT] = instr_jmp_export,
10963         [INSTR_JMP_LT_MH] = instr_jmp_export,
10964         [INSTR_JMP_LT_HM] = instr_jmp_export,
10965         [INSTR_JMP_LT_HH] = instr_jmp_export,
10966         [INSTR_JMP_LT_MI] = instr_jmp_export,
10967         [INSTR_JMP_LT_HI] = instr_jmp_export,
10968
10969         [INSTR_JMP_GT] = instr_jmp_export,
10970         [INSTR_JMP_GT_MH] = instr_jmp_export,
10971         [INSTR_JMP_GT_HM] = instr_jmp_export,
10972         [INSTR_JMP_GT_HH] = instr_jmp_export,
10973         [INSTR_JMP_GT_MI] = instr_jmp_export,
10974         [INSTR_JMP_GT_HI] = instr_jmp_export,
10975
10976         [INSTR_RETURN] = instr_return_export,
10977 };
10978
10979 static void
10980 action_data_codegen(struct action *a, FILE *f)
10981 {
10982         uint32_t i;
10983
10984         fprintf(f,
10985                 "static const struct instruction action_%s_instructions[] = {\n",
10986                 a->name);
10987
10988         for (i = 0; i < a->n_instructions; i++) {
10989                 struct instruction *instr = &a->instructions[i];
10990                 instruction_export_t func = export_table[instr->type];
10991
10992                 func(instr, f);
10993         }
10994
10995         fprintf(f, "};\n");
10996 }
10997
10998 static const char *
10999 instr_type_to_func(struct instruction *instr)
11000 {
11001         switch (instr->type) {
11002         case INSTR_RX: return NULL;
11003
11004         case INSTR_TX: return "__instr_tx_exec";
11005         case INSTR_TX_I: return "__instr_tx_i_exec";
11006         case INSTR_DROP: return "__instr_drop_exec";
11007
11008         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11009         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11010         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11011         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11012         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11013         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11014         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11015         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11016
11017         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11018
11019         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11020
11021         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11022         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11023         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11024         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11025         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11026         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11027         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11028         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11029         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11030
11031         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11032         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11033
11034         case INSTR_MOV: return "__instr_mov_exec";
11035         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11036         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11037         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11038         case INSTR_MOV_I: return "__instr_mov_i_exec";
11039
11040         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11041         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11042         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11043         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11044         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11045         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11046         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11047         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11048
11049         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11050         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11051         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11052         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11053         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11054         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11055
11056         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11057         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11058         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11059         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11060         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11061         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11062
11063         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11064         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11065         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11066         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11067
11068         case INSTR_ALU_AND: return "__instr_alu_and_exec";
11069         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11070         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11071         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11072         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11073
11074         case INSTR_ALU_OR: return "__instr_alu_or_exec";
11075         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11076         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11077         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11078         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11079
11080         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11081         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11082         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11083         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11084         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11085
11086         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11087         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11088         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11089         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11090         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11091         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11092
11093         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11094         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11095         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11096         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11097         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11098         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11099
11100         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11101         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11102         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11103
11104         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11105         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11106         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11107         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11108         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11109         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11110
11111         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11112         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11113         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11114         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11115         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11116         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11117         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11118         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11119         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11120
11121         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11122         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11123         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11124         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11125         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11126         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11127         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11128         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11129         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11130
11131         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11132         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11133         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11134
11135         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11136         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11137         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11138         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11139         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11140         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11141         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11142         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11143         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11144         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11145         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11146         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11147
11148         case INSTR_TABLE: return NULL;
11149         case INSTR_TABLE_AF: return NULL;
11150         case INSTR_SELECTOR: return NULL;
11151         case INSTR_LEARNER: return NULL;
11152         case INSTR_LEARNER_AF: return NULL;
11153
11154         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11155         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11156
11157         case INSTR_EXTERN_OBJ: return NULL;
11158         case INSTR_EXTERN_FUNC: return NULL;
11159
11160         case INSTR_JMP: return NULL;
11161         case INSTR_JMP_VALID: return NULL;
11162         case INSTR_JMP_INVALID: return NULL;
11163         case INSTR_JMP_HIT: return NULL;
11164         case INSTR_JMP_MISS: return NULL;
11165         case INSTR_JMP_ACTION_HIT: return NULL;
11166         case INSTR_JMP_ACTION_MISS: return NULL;
11167         case INSTR_JMP_EQ: return NULL;
11168         case INSTR_JMP_EQ_MH: return NULL;
11169         case INSTR_JMP_EQ_HM: return NULL;
11170         case INSTR_JMP_EQ_HH: return NULL;
11171         case INSTR_JMP_EQ_I: return NULL;
11172         case INSTR_JMP_NEQ: return NULL;
11173         case INSTR_JMP_NEQ_MH: return NULL;
11174         case INSTR_JMP_NEQ_HM: return NULL;
11175         case INSTR_JMP_NEQ_HH: return NULL;
11176         case INSTR_JMP_NEQ_I: return NULL;
11177         case INSTR_JMP_LT: return NULL;
11178         case INSTR_JMP_LT_MH: return NULL;
11179         case INSTR_JMP_LT_HM: return NULL;
11180         case INSTR_JMP_LT_HH: return NULL;
11181         case INSTR_JMP_LT_MI: return NULL;
11182         case INSTR_JMP_LT_HI: return NULL;
11183         case INSTR_JMP_GT: return NULL;
11184         case INSTR_JMP_GT_MH: return NULL;
11185         case INSTR_JMP_GT_HM: return NULL;
11186         case INSTR_JMP_GT_HH: return NULL;
11187         case INSTR_JMP_GT_MI: return NULL;
11188         case INSTR_JMP_GT_HI: return NULL;
11189
11190         case INSTR_RETURN: return NULL;
11191
11192         default: return NULL;
11193         }
11194 }
11195
11196 static void
11197 action_instr_does_tx_codegen(struct action *a,
11198                         uint32_t instr_pos,
11199                         struct instruction *instr,
11200                         FILE *f)
11201 {
11202         fprintf(f,
11203                 "%s(p, t, &action_%s_instructions[%u]);\n"
11204                 "\tthread_ip_reset(p, t);\n"
11205                 "\tinstr_rx_exec(p);\n"
11206                 "\treturn;\n",
11207                 instr_type_to_func(instr),
11208                 a->name,
11209                 instr_pos);
11210 }
11211
11212 static void
11213 action_instr_extern_obj_codegen(struct action *a,
11214                                 uint32_t instr_pos,
11215                                 FILE *f)
11216 {
11217         fprintf(f,
11218                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11219                 a->name,
11220                 instr_pos);
11221 }
11222
11223 static void
11224 action_instr_extern_func_codegen(struct action *a,
11225                                  uint32_t instr_pos,
11226                                  FILE *f)
11227 {
11228         fprintf(f,
11229                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11230                 a->name,
11231                 instr_pos);
11232 }
11233
11234 static void
11235 action_instr_jmp_codegen(struct action *a,
11236                          uint32_t instr_pos,
11237                          struct instruction *instr,
11238                          struct instruction_data *data,
11239                          FILE *f)
11240 {
11241         switch (instr->type) {
11242         case INSTR_JMP:
11243                 fprintf(f,
11244                         "goto %s;\n",
11245                         data->jmp_label);
11246                 return;
11247
11248         case INSTR_JMP_VALID:
11249                 fprintf(f,
11250                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11251                         "\t\tgoto %s;\n",
11252                         a->name,
11253                         instr_pos,
11254                         data->jmp_label);
11255                 return;
11256
11257         case INSTR_JMP_INVALID:
11258                 fprintf(f,
11259                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11260                         "\t\tgoto %s;\n",
11261                         a->name,
11262                         instr_pos,
11263                         data->jmp_label);
11264                 return;
11265
11266         case INSTR_JMP_HIT:
11267                 fprintf(f,
11268                         "if (t->hit)\n"
11269                         "\t\tgoto %s;\n",
11270                         data->jmp_label);
11271                 return;
11272
11273         case INSTR_JMP_MISS:
11274                 fprintf(f,
11275                         "if (!t->hit)\n"
11276                         "\t\tgoto %s;\n",
11277                         data->jmp_label);
11278                 return;
11279
11280         case INSTR_JMP_ACTION_HIT:
11281                 fprintf(f,
11282                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11283                         "\t\tgoto %s;\n",
11284                         a->name,
11285                         instr_pos,
11286                         data->jmp_label);
11287                 return;
11288
11289         case INSTR_JMP_ACTION_MISS:
11290                 fprintf(f,
11291                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11292                         "\t\tgoto %s;\n",
11293                         a->name,
11294                         instr_pos,
11295                         data->jmp_label);
11296                 return;
11297
11298         case INSTR_JMP_EQ:
11299                 fprintf(f,
11300                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11301                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11302                         "\t\tgoto %s;\n",
11303                         a->name,
11304                         instr_pos,
11305                         a->name,
11306                         instr_pos,
11307                         data->jmp_label);
11308                 return;
11309
11310         case INSTR_JMP_EQ_MH:
11311                 fprintf(f,
11312                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11313                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11314                         "\t\tgoto %s;\n",
11315                         a->name,
11316                         instr_pos,
11317                         a->name,
11318                         instr_pos,
11319                         data->jmp_label);
11320                 return;
11321
11322         case INSTR_JMP_EQ_HM:
11323                 fprintf(f,
11324                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11325                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11326                         "\t\tgoto %s;\n",
11327                         a->name,
11328                         instr_pos,
11329                         a->name,
11330                         instr_pos,
11331                         data->jmp_label);
11332                 return;
11333
11334         case INSTR_JMP_EQ_HH:
11335                 fprintf(f,
11336                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11337                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11338                         "\t\tgoto %s;\n",
11339                         a->name,
11340                         instr_pos,
11341                         a->name,
11342                         instr_pos,
11343                         data->jmp_label);
11344                 return;
11345
11346         case INSTR_JMP_EQ_I:
11347                 fprintf(f,
11348                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11349                         "action_%s_instructions[%u].jmp.b_val)\n"
11350                         "\t\tgoto %s;\n",
11351                         a->name,
11352                         instr_pos,
11353                         a->name,
11354                         instr_pos,
11355                         data->jmp_label);
11356                 return;
11357
11358         case INSTR_JMP_NEQ:
11359                 fprintf(f,
11360                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11361                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11362                         "\t\tgoto %s;\n",
11363                         a->name,
11364                         instr_pos,
11365                         a->name,
11366                         instr_pos,
11367                         data->jmp_label);
11368                 return;
11369
11370         case INSTR_JMP_NEQ_MH:
11371                 fprintf(f,
11372                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11373                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11374                         "\t\tgoto %s;\n",
11375                         a->name,
11376                         instr_pos,
11377                         a->name,
11378                         instr_pos,
11379                         data->jmp_label);
11380                 return;
11381
11382         case INSTR_JMP_NEQ_HM:
11383                 fprintf(f,
11384                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11385                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11386                         "\t\tgoto %s;\n",
11387                         a->name,
11388                         instr_pos,
11389                         a->name,
11390                         instr_pos,
11391                         data->jmp_label);
11392                 return;
11393
11394         case INSTR_JMP_NEQ_HH:
11395                 fprintf(f,
11396                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11397                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11398                         "\t\tgoto %s;\n",
11399                         a->name,
11400                         instr_pos,
11401                         a->name,
11402                         instr_pos,
11403                         data->jmp_label);
11404                 return;
11405
11406         case INSTR_JMP_NEQ_I:
11407                 fprintf(f,
11408                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11409                         "action_%s_instructions[%u].jmp.b_val)\n"
11410                         "\t\tgoto %s;\n",
11411                         a->name,
11412                         instr_pos,
11413                         a->name,
11414                         instr_pos,
11415                         data->jmp_label);
11416                 return;
11417
11418         case INSTR_JMP_LT:
11419                 fprintf(f,
11420                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11421                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11422                         "\t\tgoto %s;\n",
11423                         a->name,
11424                         instr_pos,
11425                         a->name,
11426                         instr_pos,
11427                         data->jmp_label);
11428                 return;
11429
11430         case INSTR_JMP_LT_MH:
11431                 fprintf(f,
11432                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11433                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11434                         "\t\tgoto %s;\n",
11435                         a->name,
11436                         instr_pos,
11437                         a->name,
11438                         instr_pos,
11439                         data->jmp_label);
11440                 return;
11441
11442         case INSTR_JMP_LT_HM:
11443                 fprintf(f,
11444                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11445                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11446                         "\t\tgoto %s;\n",
11447                         a->name,
11448                         instr_pos,
11449                         a->name,
11450                         instr_pos,
11451                         data->jmp_label);
11452                 return;
11453
11454         case INSTR_JMP_LT_HH:
11455                 fprintf(f,
11456                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11457                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11458                         "\t\tgoto %s;\n",
11459                         a->name,
11460                         instr_pos,
11461                         a->name,
11462                         instr_pos,
11463                         data->jmp_label);
11464                 return;
11465
11466         case INSTR_JMP_LT_MI:
11467                 fprintf(f,
11468                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11469                         "action_%s_instructions[%u].jmp.b_val)\n"
11470                         "\t\tgoto %s;\n",
11471                         a->name,
11472                         instr_pos,
11473                         a->name,
11474                         instr_pos,
11475                         data->jmp_label);
11476                 return;
11477
11478         case INSTR_JMP_LT_HI:
11479                 fprintf(f,
11480                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11481                         "action_%s_instructions[%u].jmp.b_val)\n"
11482                         "\t\tgoto %s;\n",
11483                         a->name,
11484                         instr_pos,
11485                         a->name,
11486                         instr_pos,
11487                         data->jmp_label);
11488                 return;
11489
11490         case INSTR_JMP_GT:
11491                 fprintf(f,
11492                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11493                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11494                         "\t\tgoto %s;\n",
11495                         a->name,
11496                         instr_pos,
11497                         a->name,
11498                         instr_pos,
11499                         data->jmp_label);
11500                 return;
11501
11502         case INSTR_JMP_GT_MH:
11503                 fprintf(f,
11504                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11505                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11506                         "\t\tgoto %s;\n",
11507                         a->name,
11508                         instr_pos,
11509                         a->name,
11510                         instr_pos,
11511                         data->jmp_label);
11512                 return;
11513
11514         case INSTR_JMP_GT_HM:
11515                 fprintf(f,
11516                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11517                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11518                         "\t\tgoto %s;\n",
11519                         a->name,
11520                         instr_pos,
11521                         a->name,
11522                         instr_pos,
11523                         data->jmp_label);
11524                 return;
11525
11526         case INSTR_JMP_GT_HH:
11527                 fprintf(f,
11528                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11529                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11530                         "\t\tgoto %s;\n",
11531                         a->name,
11532                         instr_pos,
11533                         a->name,
11534                         instr_pos,
11535                         data->jmp_label);
11536                 return;
11537
11538         case INSTR_JMP_GT_MI:
11539                 fprintf(f,
11540                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11541                         "action_%s_instructions[%u].jmp.b_val)\n"
11542                         "\t\tgoto %s;\n",
11543                         a->name,
11544                         instr_pos,
11545                         a->name,
11546                         instr_pos,
11547                         data->jmp_label);
11548                 return;
11549
11550         case INSTR_JMP_GT_HI:
11551                 fprintf(f,
11552                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11553                         "action_%s_instructions[%u].jmp.b_val)\n"
11554                         "\t\tgoto %s;\n",
11555                         a->name,
11556                         instr_pos,
11557                         a->name,
11558                         instr_pos,
11559                         data->jmp_label);
11560                 return;
11561
11562         default:
11563                 return;
11564         }
11565 }
11566
11567 static void
11568 action_instr_return_codegen(FILE *f)
11569 {
11570         fprintf(f,
11571                 "return;\n");
11572 }
11573
11574 static void
11575 action_instr_codegen(struct action *a, FILE *f)
11576 {
11577         uint32_t i;
11578
11579         fprintf(f,
11580                 "void\n"
11581                 "action_%s_run(struct rte_swx_pipeline *p)\n"
11582                 "{\n"
11583                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11584                 "\n",
11585                 a->name);
11586
11587         for (i = 0; i < a->n_instructions; i++) {
11588                 struct instruction *instr = &a->instructions[i];
11589                 struct instruction_data *data = &a->instruction_data[i];
11590
11591                 /* Label, if present. */
11592                 if (data->label[0])
11593                         fprintf(f, "\n%s : ", data->label);
11594                 else
11595                         fprintf(f, "\n\t");
11596
11597                 /* TX instruction type. */
11598                 if (instruction_does_tx(instr)) {
11599                         action_instr_does_tx_codegen(a, i, instr, f);
11600                         continue;
11601                 }
11602
11603                 /* Extern object/function instruction type. */
11604                 if (instr->type == INSTR_EXTERN_OBJ) {
11605                         action_instr_extern_obj_codegen(a, i, f);
11606                         continue;
11607                 }
11608
11609                 if (instr->type == INSTR_EXTERN_FUNC) {
11610                         action_instr_extern_func_codegen(a, i, f);
11611                         continue;
11612                 }
11613
11614                 /* Jump instruction type. */
11615                 if (instruction_is_jmp(instr)) {
11616                         action_instr_jmp_codegen(a, i, instr, data, f);
11617                         continue;
11618                 }
11619
11620                 /* Return instruction type. */
11621                 if (instr->type == INSTR_RETURN) {
11622                         action_instr_return_codegen(f);
11623                         continue;
11624                 }
11625
11626                 /* Any other instruction type. */
11627                 fprintf(f,
11628                         "%s(p, t, &action_%s_instructions[%u]);\n",
11629                         instr_type_to_func(instr),
11630                         a->name,
11631                         i);
11632         }
11633
11634         fprintf(f, "}\n\n");
11635 }
11636
11637 struct instruction_group {
11638         TAILQ_ENTRY(instruction_group) node;
11639
11640         uint32_t group_id;
11641
11642         uint32_t first_instr_id;
11643
11644         uint32_t last_instr_id;
11645
11646         instr_exec_t func;
11647 };
11648
11649 TAILQ_HEAD(instruction_group_list, instruction_group);
11650
11651 static struct instruction_group *
11652 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11653 {
11654         struct instruction_group *g;
11655
11656         TAILQ_FOREACH(g, igl, node)
11657                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11658                         return g;
11659
11660         return NULL;
11661 }
11662
11663 static void
11664 instruction_group_list_free(struct instruction_group_list *igl)
11665 {
11666         if (!igl)
11667                 return;
11668
11669         for ( ; ; ) {
11670                 struct instruction_group *g;
11671
11672                 g = TAILQ_FIRST(igl);
11673                 if (!g)
11674                         break;
11675
11676                 TAILQ_REMOVE(igl, g, node);
11677                 free(g);
11678         }
11679
11680         free(igl);
11681 }
11682
11683 static struct instruction_group_list *
11684 instruction_group_list_create(struct rte_swx_pipeline *p)
11685 {
11686         struct instruction_group_list *igl = NULL;
11687         struct instruction_group *g = NULL;
11688         uint32_t n_groups = 0, i;
11689
11690         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11691                 goto error;
11692
11693         /* List init. */
11694         igl = calloc(1, sizeof(struct instruction_group_list));
11695         if (!igl)
11696                 goto error;
11697
11698         TAILQ_INIT(igl);
11699
11700         /* Allocate the first group. */
11701         g = calloc(1, sizeof(struct instruction_group));
11702         if (!g)
11703                 goto error;
11704
11705         /* Iteration 1: Separate the instructions into groups based on the thread yield
11706          * instructions. Do not worry about the jump instructions at this point.
11707          */
11708         for (i = 0; i < p->n_instructions; i++) {
11709                 struct instruction *instr = &p->instructions[i];
11710
11711                 /* Check for thread yield instructions. */
11712                 if (!instruction_does_thread_yield(instr))
11713                         continue;
11714
11715                 /* If the current group contains at least one instruction, then finalize it (with
11716                  * the previous instruction), add it to the list and allocate a new group (that
11717                  * starts with the current instruction).
11718                  */
11719                 if (i - g->first_instr_id) {
11720                         /* Finalize the group. */
11721                         g->last_instr_id = i - 1;
11722
11723                         /* Add the group to the list. Advance the number of groups. */
11724                         TAILQ_INSERT_TAIL(igl, g, node);
11725                         n_groups++;
11726
11727                         /* Allocate a new group. */
11728                         g = calloc(1, sizeof(struct instruction_group));
11729                         if (!g)
11730                                 goto error;
11731
11732                         /* Initialize the new group. */
11733                         g->group_id = n_groups;
11734                         g->first_instr_id = i;
11735                 }
11736
11737                 /* Finalize the current group (with the current instruction, therefore this group
11738                  * contains just the current thread yield instruction), add it to the list and
11739                  * allocate a new group (that starts with the next instruction).
11740                  */
11741
11742                 /* Finalize the group. */
11743                 g->last_instr_id = i;
11744
11745                 /* Add the group to the list. Advance the number of groups. */
11746                 TAILQ_INSERT_TAIL(igl, g, node);
11747                 n_groups++;
11748
11749                 /* Allocate a new group. */
11750                 g = calloc(1, sizeof(struct instruction_group));
11751                 if (!g)
11752                         goto error;
11753
11754                 /* Initialize the new group. */
11755                 g->group_id = n_groups;
11756                 g->first_instr_id = i + 1;
11757         }
11758
11759         /* Handle the last group. */
11760         if (i - g->first_instr_id) {
11761                 /* Finalize the group. */
11762                 g->last_instr_id = i - 1;
11763
11764                 /* Add the group to the list. Advance the number of groups. */
11765                 TAILQ_INSERT_TAIL(igl, g, node);
11766                 n_groups++;
11767         } else
11768                 free(g);
11769
11770         g = NULL;
11771
11772         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11773          * the destination of a jump instruction located in a different group ("far jump"), then the
11774          * current group has to be split, so that the instruction representing the far jump
11775          * destination is at the start of its group.
11776          */
11777         for ( ; ; ) {
11778                 int is_modified = 0;
11779
11780                 for (i = 0; i < p->n_instructions; i++) {
11781                         struct instruction_data *data = &p->instruction_data[i];
11782                         struct instruction_group *g;
11783                         uint32_t j;
11784
11785                         /* Continue when the current instruction is not a jump destination. */
11786                         if (!data->n_users)
11787                                 continue;
11788
11789                         g = instruction_group_list_group_find(igl, i);
11790                         if (!g)
11791                                 goto error;
11792
11793                         /* Find out all the jump instructions with this destination. */
11794                         for (j = 0; j < p->n_instructions; j++) {
11795                                 struct instruction *jmp_instr = &p->instructions[j];
11796                                 struct instruction_data *jmp_data = &p->instruction_data[j];
11797                                 struct instruction_group *jmp_g, *new_g;
11798
11799                                 /* Continue when not a jump instruction. Even when jump instruction,
11800                                  * continue when the jump destination is not this instruction.
11801                                  */
11802                                 if (!instruction_is_jmp(jmp_instr) ||
11803                                     strcmp(jmp_data->jmp_label, data->label))
11804                                         continue;
11805
11806                                 jmp_g = instruction_group_list_group_find(igl, j);
11807                                 if (!jmp_g)
11808                                         goto error;
11809
11810                                 /* Continue when both the jump instruction and the jump destination
11811                                  * instruction are in the same group. Even when in different groups,
11812                                  * still continue if the jump destination instruction is already the
11813                                  * first instruction of its group.
11814                                  */
11815                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11816                                         continue;
11817
11818                                 /* Split the group of the current jump destination instruction to
11819                                  * make this instruction the first instruction of a new group.
11820                                  */
11821                                 new_g = calloc(1, sizeof(struct instruction_group));
11822                                 if (!new_g)
11823                                         goto error;
11824
11825                                 new_g->group_id = n_groups;
11826                                 new_g->first_instr_id = i;
11827                                 new_g->last_instr_id = g->last_instr_id;
11828
11829                                 g->last_instr_id = i - 1;
11830
11831                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11832                                 n_groups++;
11833                                 is_modified = 1;
11834
11835                                 /* The decision to split this group (to make the current instruction
11836                                  * the first instruction of a new group) is already taken and fully
11837                                  * implemented, so no need to search for more reasons to do it.
11838                                  */
11839                                 break;
11840                         }
11841                 }
11842
11843                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11844                  * previously considered local (i.e. the jump destination is in the same group as
11845                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11846                  * different group than the jump instruction). Wost case scenario: each instruction
11847                  * that is a jump destination ends up as the first instruction of its group.
11848                  */
11849                 if (!is_modified)
11850                         break;
11851         }
11852
11853         /* Re-assign the group IDs to be in incremental order. */
11854         i = 0;
11855         TAILQ_FOREACH(g, igl, node) {
11856                 g->group_id = i;
11857
11858                 i++;
11859         }
11860
11861         return igl;
11862
11863 error:
11864         instruction_group_list_free(igl);
11865
11866         free(g);
11867
11868         return NULL;
11869 }
11870
11871 static void
11872 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11873                                uint32_t instr_pos,
11874                                struct instruction *instr,
11875                                FILE *f)
11876 {
11877         fprintf(f,
11878                 "%s(p, t, &pipeline_instructions[%u]);\n"
11879                 "\tthread_ip_reset(p, t);\n"
11880                 "\tinstr_rx_exec(p);\n"
11881                 "\treturn;\n",
11882                 instr_type_to_func(instr),
11883                 instr_pos);
11884 }
11885
11886 static int
11887 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11888                            struct instruction_group_list *igl,
11889                            uint32_t jmp_instr_id,
11890                            struct instruction *jmp_instr,
11891                            struct instruction_data *jmp_data,
11892                            FILE *f)
11893 {
11894         struct instruction_group *jmp_g, *g;
11895         struct instruction_data *data;
11896         uint32_t instr_id;
11897
11898         switch (jmp_instr->type) {
11899         case INSTR_JMP:
11900                 break;
11901
11902         case INSTR_JMP_VALID:
11903                 fprintf(f,
11904                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11905                         jmp_instr_id);
11906                 break;
11907
11908         case INSTR_JMP_INVALID:
11909                 fprintf(f,
11910                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11911                         jmp_instr_id);
11912                 break;
11913
11914         case INSTR_JMP_HIT:
11915                 fprintf(f,
11916                         "if (t->hit)\n");
11917                 break;
11918
11919         case INSTR_JMP_MISS:
11920                 fprintf(f,
11921                         "if (!t->hit)\n");
11922                 break;
11923
11924         case INSTR_JMP_ACTION_HIT:
11925                 fprintf(f,
11926                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
11927                         jmp_instr_id);
11928                 break;
11929
11930         case INSTR_JMP_ACTION_MISS:
11931                 fprintf(f,
11932                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
11933                         jmp_instr_id);
11934                 break;
11935
11936         case INSTR_JMP_EQ:
11937                 fprintf(f,
11938                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11939                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11940                         jmp_instr_id,
11941                         jmp_instr_id);
11942                 break;
11943
11944         case INSTR_JMP_EQ_MH:
11945                 fprintf(f,
11946                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11947                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11948                         jmp_instr_id,
11949                         jmp_instr_id);
11950                 break;
11951
11952         case INSTR_JMP_EQ_HM:
11953                 fprintf(f,
11954                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11955                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11956                         jmp_instr_id,
11957                         jmp_instr_id);
11958                 break;
11959
11960         case INSTR_JMP_EQ_HH:
11961                 fprintf(f,
11962                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11963                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11964                         jmp_instr_id,
11965                         jmp_instr_id);
11966                 break;
11967
11968         case INSTR_JMP_EQ_I:
11969                 fprintf(f,
11970                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11971                         "pipeline_instructions[%u].jmp.b_val)",
11972                         jmp_instr_id,
11973                         jmp_instr_id);
11974                 break;
11975
11976         case INSTR_JMP_NEQ:
11977                 fprintf(f,
11978                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11979                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11980                         jmp_instr_id,
11981                         jmp_instr_id);
11982                 break;
11983
11984         case INSTR_JMP_NEQ_MH:
11985                 fprintf(f,
11986                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11987                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11988                         jmp_instr_id,
11989                         jmp_instr_id);
11990                 break;
11991
11992         case INSTR_JMP_NEQ_HM:
11993                 fprintf(f,
11994                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11995                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11996                         jmp_instr_id,
11997                         jmp_instr_id);
11998                 break;
11999
12000         case INSTR_JMP_NEQ_HH:
12001                 fprintf(f,
12002                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12003                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12004                         jmp_instr_id,
12005                         jmp_instr_id);
12006                 break;
12007
12008         case INSTR_JMP_NEQ_I:
12009                 fprintf(f,
12010                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12011                         "pipeline_instructions[%u].jmp.b_val)",
12012                         jmp_instr_id,
12013                         jmp_instr_id);
12014                 break;
12015
12016         case INSTR_JMP_LT:
12017                 fprintf(f,
12018                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12019                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12020                         jmp_instr_id,
12021                         jmp_instr_id);
12022                 break;
12023
12024         case INSTR_JMP_LT_MH:
12025                 fprintf(f,
12026                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12027                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12028                         jmp_instr_id,
12029                         jmp_instr_id);
12030                 break;
12031
12032         case INSTR_JMP_LT_HM:
12033                 fprintf(f,
12034                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12035                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12036                         jmp_instr_id,
12037                         jmp_instr_id);
12038                 break;
12039
12040         case INSTR_JMP_LT_HH:
12041                 fprintf(f,
12042                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12043                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12044                         jmp_instr_id,
12045                         jmp_instr_id);
12046                 break;
12047
12048         case INSTR_JMP_LT_MI:
12049                 fprintf(f,
12050                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12051                         "pipeline_instructions[%u].jmp.b_val)",
12052                         jmp_instr_id,
12053                         jmp_instr_id);
12054                 break;
12055
12056         case INSTR_JMP_LT_HI:
12057                 fprintf(f,
12058                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12059                         "pipeline_instructions[%u].jmp.b_val)",
12060                         jmp_instr_id,
12061                         jmp_instr_id);
12062                 break;
12063
12064         case INSTR_JMP_GT:
12065                 fprintf(f,
12066                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12067                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12068                         jmp_instr_id,
12069                         jmp_instr_id);
12070                 break;
12071
12072         case INSTR_JMP_GT_MH:
12073                 fprintf(f,
12074                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12075                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12076                         jmp_instr_id,
12077                         jmp_instr_id);
12078                 break;
12079
12080         case INSTR_JMP_GT_HM:
12081                 fprintf(f,
12082                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12083                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12084                         jmp_instr_id,
12085                         jmp_instr_id);
12086                 break;
12087
12088         case INSTR_JMP_GT_HH:
12089                 fprintf(f,
12090                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12091                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12092                         jmp_instr_id,
12093                         jmp_instr_id);
12094                 break;
12095
12096         case INSTR_JMP_GT_MI:
12097                 fprintf(f,
12098                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12099                         "pipeline_instructions[%u].jmp.b_val)",
12100                         jmp_instr_id,
12101                         jmp_instr_id);
12102                 break;
12103
12104         case INSTR_JMP_GT_HI:
12105                 fprintf(f,
12106                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12107                         "pipeline_instructions[%u].jmp.b_val)",
12108                         jmp_instr_id,
12109                         jmp_instr_id);
12110                 break;
12111
12112         default:
12113                 break;
12114         }
12115
12116         /* Find the instruction group of the jump instruction. */
12117         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12118         if (!jmp_g)
12119                 return -EINVAL;
12120
12121         /* Find the instruction group of the jump destination instruction. */
12122         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12123         if (!data)
12124                 return -EINVAL;
12125
12126         instr_id = data - p->instruction_data;
12127
12128         g = instruction_group_list_group_find(igl, instr_id);
12129         if (!g)
12130                 return -EINVAL;
12131
12132         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12133          * instruction group).
12134          */
12135         if (g->group_id == jmp_g->group_id)
12136                 fprintf(f,
12137                         "\n\t\tgoto %s;\n",
12138                         jmp_data->jmp_label);
12139         else
12140                 fprintf(f,
12141                         " {\n"
12142                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12143                         "\t\treturn;\n"
12144                         "\t}\n\n",
12145                         g->group_id);
12146
12147         return 0;
12148 }
12149
12150 static void
12151 instruction_group_list_codegen(struct instruction_group_list *igl,
12152                                struct rte_swx_pipeline *p,
12153                                FILE *f)
12154 {
12155         struct instruction_group *g;
12156         uint32_t i;
12157         int is_required = 0;
12158
12159         /* Check if code generation is required. */
12160         TAILQ_FOREACH(g, igl, node)
12161                 if (g->first_instr_id < g->last_instr_id)
12162                         is_required = 1;
12163
12164         if (!is_required)
12165                 return;
12166
12167         /* Generate the code for the pipeline instruction array. */
12168         fprintf(f,
12169                 "static const struct instruction pipeline_instructions[] = {\n");
12170
12171         for (i = 0; i < p->n_instructions; i++) {
12172                 struct instruction *instr = &p->instructions[i];
12173                 instruction_export_t func = export_table[instr->type];
12174
12175                 func(instr, f);
12176         }
12177
12178         fprintf(f, "};\n\n");
12179
12180         /* Generate the code for the pipeline functions: one function for each instruction group
12181          * that contains more than one instruction.
12182          */
12183         TAILQ_FOREACH(g, igl, node) {
12184                 struct instruction *last_instr;
12185                 uint32_t j;
12186
12187                 /* Skip if group contains a single instruction. */
12188                 if (g->last_instr_id == g->first_instr_id)
12189                         continue;
12190
12191                 /* Generate new pipeline function. */
12192                 fprintf(f,
12193                         "void\n"
12194                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12195                         "{\n"
12196                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12197                         "\n",
12198                         g->group_id);
12199
12200                 /* Generate the code for each pipeline instruction. */
12201                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12202                         struct instruction *instr = &p->instructions[j];
12203                         struct instruction_data *data = &p->instruction_data[j];
12204
12205                         /* Label, if present. */
12206                         if (data->label[0])
12207                                 fprintf(f, "\n%s : ", data->label);
12208                         else
12209                                 fprintf(f, "\n\t");
12210
12211                         /* TX instruction type. */
12212                         if (instruction_does_tx(instr)) {
12213                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12214                                 continue;
12215                         }
12216
12217                         /* Jump instruction type. */
12218                         if (instruction_is_jmp(instr)) {
12219                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12220                                 continue;
12221                         }
12222
12223                         /* Any other instruction type. */
12224                         fprintf(f,
12225                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12226                                 instr_type_to_func(instr),
12227                                 j);
12228                 }
12229
12230                 /* Finalize the generated pipeline function. For some instructions such as TX,
12231                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12232                  * decided unconditionally and the instruction pointer of the current thread set
12233                  * accordingly; for all the other instructions, the instruction pointer must be
12234                  * incremented now.
12235                  */
12236                 last_instr = &p->instructions[g->last_instr_id];
12237
12238                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12239                         fprintf(f,
12240                                 "thread_ip_inc(p);\n");
12241
12242                 fprintf(f,
12243                         "}\n"
12244                         "\n");
12245         }
12246 }
12247
12248 static uint32_t
12249 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12250 {
12251         struct instruction_group *g;
12252         uint32_t n_custom_instr = 0;
12253
12254         /* Groups with a single instruction: no function is generated for this group, the group
12255          * keeps its current instruction. Groups with more than two instructions: one function and
12256          * the associated custom instruction get generated for each such group.
12257          */
12258         TAILQ_FOREACH(g, igl, node) {
12259                 if (g->first_instr_id == g->last_instr_id)
12260                         continue;
12261
12262                 n_custom_instr++;
12263         }
12264
12265         return n_custom_instr;
12266 }
12267
12268 static int
12269 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12270 {
12271         struct action *a;
12272         FILE *f = NULL;
12273
12274         /* Create the .c file. */
12275         f = fopen("/tmp/pipeline.c", "w");
12276         if (!f)
12277                 return -EIO;
12278
12279         /* Include the .h file. */
12280         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12281
12282         /* Add the code for each action. */
12283         TAILQ_FOREACH(a, &p->actions, node) {
12284                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12285
12286                 action_data_codegen(a, f);
12287
12288                 fprintf(f, "\n");
12289
12290                 action_instr_codegen(a, f);
12291
12292                 fprintf(f, "\n");
12293         }
12294
12295         /* Add the pipeline code. */
12296         instruction_group_list_codegen(igl, p, f);
12297
12298         /* Close the .c file. */
12299         fclose(f);
12300
12301         return 0;
12302 }
12303
12304 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12305 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12306 #endif
12307
12308 static int
12309 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12310 {
12311         struct action *a;
12312         struct instruction_group *g;
12313         char *dir_in, *buffer = NULL;
12314         const char *dir_out;
12315         int status = 0;
12316
12317         /* Get the environment variables. */
12318         dir_in = getenv("RTE_INSTALL_DIR");
12319         if (!dir_in) {
12320                 status = -EINVAL;
12321                 goto free;
12322         }
12323
12324         dir_out = "/tmp";
12325
12326         /* Memory allocation for the command buffer. */
12327         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12328         if (!buffer) {
12329                 status = -ENOMEM;
12330                 goto free;
12331         }
12332
12333         snprintf(buffer,
12334                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12335                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12336                  "-I %s/lib/pipeline "
12337                  "-I %s/lib/eal/include "
12338                  "-I %s/lib/eal/x86/include "
12339                  "-I %s/lib/eal/include/generic "
12340                  "-I %s/lib/meter "
12341                  "-I %s/lib/port "
12342                  "-I %s/lib/table "
12343                  "-I %s/lib/pipeline "
12344                  "-I %s/config "
12345                  "-I %s/build "
12346                  "-I %s/lib/eal/linux/include "
12347                  ">%s/pipeline.log 2>&1 "
12348                  "&& "
12349                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12350                  ">>%s/pipeline.log 2>&1",
12351                  dir_out,
12352                  dir_out,
12353                  dir_in,
12354                  dir_in,
12355                  dir_in,
12356                  dir_in,
12357                  dir_in,
12358                  dir_in,
12359                  dir_in,
12360                  dir_in,
12361                  dir_in,
12362                  dir_in,
12363                  dir_in,
12364                  dir_out,
12365                  dir_out,
12366                  dir_out,
12367                  dir_out);
12368
12369         /* Build the shared object library. */
12370         status = system(buffer);
12371         if (status)
12372                 goto free;
12373
12374         /* Open library. */
12375         snprintf(buffer,
12376                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12377                  "%s/libpipeline.so",
12378                  dir_out);
12379
12380         p->lib = dlopen(buffer, RTLD_LAZY);
12381         if (!p->lib) {
12382                 status = -EIO;
12383                 goto free;
12384         }
12385
12386         /* Get the action function symbols. */
12387         TAILQ_FOREACH(a, &p->actions, node) {
12388                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12389
12390                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12391                 if (!p->action_funcs[a->id]) {
12392                         status = -EINVAL;
12393                         goto free;
12394                 }
12395         }
12396
12397         /* Get the pipeline function symbols. */
12398         TAILQ_FOREACH(g, igl, node) {
12399                 if (g->first_instr_id == g->last_instr_id)
12400                         continue;
12401
12402                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12403
12404                 g->func = dlsym(p->lib, buffer);
12405                 if (!g->func) {
12406                         status = -EINVAL;
12407                         goto free;
12408                 }
12409         }
12410
12411 free:
12412         if (status && p->lib) {
12413                 dlclose(p->lib);
12414                 p->lib = NULL;
12415         }
12416
12417         free(buffer);
12418
12419         return status;
12420 }
12421
12422 static int
12423 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12424                       struct instruction_group_list *igl)
12425 {
12426         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12427
12428         /* Check that enough space is available within the pipeline instruction table to store all
12429          * the custom instructions.
12430          */
12431         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12432                 return -ENOSPC;
12433
12434         return 0;
12435 }
12436
12437 static void
12438 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12439 {
12440         struct instruction_group *g;
12441         uint32_t i;
12442
12443         /* Pipeline table instructions. */
12444         for (i = 0; i < p->n_instructions; i++) {
12445                 struct instruction *instr = &p->instructions[i];
12446
12447                 if (instr->type == INSTR_TABLE)
12448                         instr->type = INSTR_TABLE_AF;
12449
12450                 if (instr->type == INSTR_LEARNER)
12451                         instr->type = INSTR_LEARNER_AF;
12452         }
12453
12454         /* Pipeline custom instructions. */
12455         i = 0;
12456         TAILQ_FOREACH(g, igl, node) {
12457                 struct instruction *instr = &p->instructions[g->first_instr_id];
12458                 uint32_t j;
12459
12460                 if (g->first_instr_id == g->last_instr_id)
12461                         continue;
12462
12463                 /* Install a new custom instruction. */
12464                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12465
12466                 /* First instruction of the group: change its type to the new custom instruction. */
12467                 instr->type = INSTR_CUSTOM_0 + i;
12468
12469                 /* All the subsequent instructions of the group: invalidate. */
12470                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12471                         struct instruction_data *data = &p->instruction_data[j];
12472
12473                         data->invalid = 1;
12474                 }
12475
12476                 i++;
12477         }
12478
12479         /* Remove the invalidated instructions. */
12480         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12481
12482         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12483          * instructions that are the only instruction within their group, so they were left
12484          * unmodified).
12485          */
12486         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12487 }
12488
12489 static int
12490 pipeline_compile(struct rte_swx_pipeline *p)
12491 {
12492         struct instruction_group_list *igl = NULL;
12493         int status = 0;
12494
12495         igl = instruction_group_list_create(p);
12496         if (!igl) {
12497                 status = -ENOMEM;
12498                 goto free;
12499         }
12500
12501         /* Code generation. */
12502         status = pipeline_codegen(p, igl);
12503         if (status)
12504                 goto free;
12505
12506         /* Build and load the shared object library. */
12507         status = pipeline_libload(p, igl);
12508         if (status)
12509                 goto free;
12510
12511         /* Adjust instructions. */
12512         status = pipeline_adjust_check(p, igl);
12513         if (status)
12514                 goto free;
12515
12516         pipeline_adjust(p, igl);
12517
12518 free:
12519         instruction_group_list_free(igl);
12520
12521         return status;
12522 }