lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8 #include <dlfcn.h>
   9
  10 #include "rte_swx_pipeline_internal.h"
  11
  12 #define CHECK(condition, err_code)                                             \
  13 do {                                                                           \
  14         if (!(condition))                                                      \
  15                 return -(err_code);                                            \
  16 } while (0)
  17
  18 #define CHECK_NAME(name, err_code)                                             \
  19         CHECK((name) &&                                                        \
  20               (name)[0] &&                                                     \
  21               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  22               err_code)
  23
  24 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  25         CHECK((instr) &&                                                       \
  26               (instr)[0] &&                                                    \
  27               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  28                RTE_SWX_INSTRUCTION_SIZE),                                      \
  29               err_code)
  30
  31 /*
  32  * Environment.
  33  */
  34 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  35
  36 #include <rte_malloc.h>
  37
  38 static void *
  39 env_malloc(size_t size, size_t alignment, int numa_node)
  40 {
  41         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  42 }
  43
  44 static void
  45 env_free(void *start, size_t size __rte_unused)
  46 {
  47         rte_free(start);
  48 }
  49
  50 #else
  51
  52 #include <numa.h>
  53
  54 static void *
  55 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  56 {
  57         void *start;
  58
  59         if (numa_available() == -1)
  60                 return NULL;
  61
  62         start = numa_alloc_onnode(size, numa_node);
  63         if (!start)
  64                 return NULL;
  65
  66         memset(start, 0, size);
  67         return start;
  68 }
  69
  70 static void
  71 env_free(void *start, size_t size)
  72 {
  73         if (numa_available() == -1)
  74                 return;
  75
  76         numa_free(start, size);
  77 }
  78
  79 #endif
  80
  81 /*
  82  * Struct.
  83  */
  84 static struct struct_type *
  85 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  86 {
  87         struct struct_type *elem;
  88
  89         TAILQ_FOREACH(elem, &p->struct_types, node)
  90                 if (strcmp(elem->name, name) == 0)
  91                         return elem;
  92
  93         return NULL;
  94 }
  95
  96 static struct field *
  97 struct_type_field_find(struct struct_type *st, const char *name)
  98 {
  99         uint32_t i;
 100
 101         for (i = 0; i < st->n_fields; i++) {
 102                 struct field *f = &st->fields[i];
 103
 104                 if (strcmp(f->name, name) == 0)
 105                         return f;
 106         }
 107
 108         return NULL;
 109 }
 110
 111 int
 112 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 113                                       const char *name,
 114                                       struct rte_swx_field_params *fields,
 115                                       uint32_t n_fields,
 116                                       int last_field_has_variable_size)
 117 {
 118         struct struct_type *st;
 119         uint32_t i;
 120
 121         CHECK(p, EINVAL);
 122         CHECK_NAME(name, EINVAL);
 123         CHECK(fields, EINVAL);
 124         CHECK(n_fields, EINVAL);
 125
 126         for (i = 0; i < n_fields; i++) {
 127                 struct rte_swx_field_params *f = &fields[i];
 128                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 129                 uint32_t j;
 130
 131                 CHECK_NAME(f->name, EINVAL);
 132                 CHECK(f->n_bits, EINVAL);
 133                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 134                 CHECK((f->n_bits & 7) == 0, EINVAL);
 135
 136                 for (j = 0; j < i; j++) {
 137                         struct rte_swx_field_params *f_prev = &fields[j];
 138
 139                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 140                 }
 141         }
 142
 143         CHECK(!struct_type_find(p, name), EEXIST);
 144
 145         /* Node allocation. */
 146         st = calloc(1, sizeof(struct struct_type));
 147         CHECK(st, ENOMEM);
 148
 149         st->fields = calloc(n_fields, sizeof(struct field));
 150         if (!st->fields) {
 151                 free(st);
 152                 CHECK(0, ENOMEM);
 153         }
 154
 155         /* Node initialization. */
 156         strcpy(st->name, name);
 157         for (i = 0; i < n_fields; i++) {
 158                 struct field *dst = &st->fields[i];
 159                 struct rte_swx_field_params *src = &fields[i];
 160                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 161
 162                 strcpy(dst->name, src->name);
 163                 dst->n_bits = src->n_bits;
 164                 dst->offset = st->n_bits;
 165                 dst->var_size = var_size;
 166
 167                 st->n_bits += src->n_bits;
 168                 st->n_bits_min += var_size ? 0 : src->n_bits;
 169         }
 170         st->n_fields = n_fields;
 171         st->var_size = last_field_has_variable_size;
 172
 173         /* Node add to tailq. */
 174         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 175
 176         return 0;
 177 }
 178
 179 static int
 180 struct_build(struct rte_swx_pipeline *p)
 181 {
 182         uint32_t i;
 183
 184         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 185                 struct thread *t = &p->threads[i];
 186
 187                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 188                 CHECK(t->structs, ENOMEM);
 189         }
 190
 191         return 0;
 192 }
 193
 194 static void
 195 struct_build_free(struct rte_swx_pipeline *p)
 196 {
 197         uint32_t i;
 198
 199         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 200                 struct thread *t = &p->threads[i];
 201
 202                 free(t->structs);
 203                 t->structs = NULL;
 204         }
 205 }
 206
 207 static void
 208 struct_free(struct rte_swx_pipeline *p)
 209 {
 210         struct_build_free(p);
 211
 212         /* Struct types. */
 213         for ( ; ; ) {
 214                 struct struct_type *elem;
 215
 216                 elem = TAILQ_FIRST(&p->struct_types);
 217                 if (!elem)
 218                         break;
 219
 220                 TAILQ_REMOVE(&p->struct_types, elem, node);
 221                 free(elem->fields);
 222                 free(elem);
 223         }
 224 }
 225
 226 /*
 227  * Input port.
 228  */
 229 static struct port_in_type *
 230 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 231 {
 232         struct port_in_type *elem;
 233
 234         if (!name)
 235                 return NULL;
 236
 237         TAILQ_FOREACH(elem, &p->port_in_types, node)
 238                 if (strcmp(elem->name, name) == 0)
 239                         return elem;
 240
 241         return NULL;
 242 }
 243
 244 int
 245 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 246                                        const char *name,
 247                                        struct rte_swx_port_in_ops *ops)
 248 {
 249         struct port_in_type *elem;
 250
 251         CHECK(p, EINVAL);
 252         CHECK_NAME(name, EINVAL);
 253         CHECK(ops, EINVAL);
 254         CHECK(ops->create, EINVAL);
 255         CHECK(ops->free, EINVAL);
 256         CHECK(ops->pkt_rx, EINVAL);
 257         CHECK(ops->stats_read, EINVAL);
 258
 259         CHECK(!port_in_type_find(p, name), EEXIST);
 260
 261         /* Node allocation. */
 262         elem = calloc(1, sizeof(struct port_in_type));
 263         CHECK(elem, ENOMEM);
 264
 265         /* Node initialization. */
 266         strcpy(elem->name, name);
 267         memcpy(&elem->ops, ops, sizeof(*ops));
 268
 269         /* Node add to tailq. */
 270         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 271
 272         return 0;
 273 }
 274
 275 static struct port_in *
 276 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 277 {
 278         struct port_in *port;
 279
 280         TAILQ_FOREACH(port, &p->ports_in, node)
 281                 if (port->id == port_id)
 282                         return port;
 283
 284         return NULL;
 285 }
 286
 287 int
 288 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 289                                 uint32_t port_id,
 290                                 const char *port_type_name,
 291                                 void *args)
 292 {
 293         struct port_in_type *type = NULL;
 294         struct port_in *port = NULL;
 295         void *obj = NULL;
 296
 297         CHECK(p, EINVAL);
 298
 299         CHECK(!port_in_find(p, port_id), EINVAL);
 300
 301         CHECK_NAME(port_type_name, EINVAL);
 302         type = port_in_type_find(p, port_type_name);
 303         CHECK(type, EINVAL);
 304
 305         obj = type->ops.create(args);
 306         CHECK(obj, ENODEV);
 307
 308         /* Node allocation. */
 309         port = calloc(1, sizeof(struct port_in));
 310         CHECK(port, ENOMEM);
 311
 312         /* Node initialization. */
 313         port->type = type;
 314         port->obj = obj;
 315         port->id = port_id;
 316
 317         /* Node add to tailq. */
 318         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 319         if (p->n_ports_in < port_id + 1)
 320                 p->n_ports_in = port_id + 1;
 321
 322         return 0;
 323 }
 324
 325 static int
 326 port_in_build(struct rte_swx_pipeline *p)
 327 {
 328         struct port_in *port;
 329         uint32_t i;
 330
 331         CHECK(p->n_ports_in, EINVAL);
 332         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 333
 334         for (i = 0; i < p->n_ports_in; i++)
 335                 CHECK(port_in_find(p, i), EINVAL);
 336
 337         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 338         CHECK(p->in, ENOMEM);
 339
 340         TAILQ_FOREACH(port, &p->ports_in, node) {
 341                 struct port_in_runtime *in = &p->in[port->id];
 342
 343                 in->pkt_rx = port->type->ops.pkt_rx;
 344                 in->obj = port->obj;
 345         }
 346
 347         return 0;
 348 }
 349
 350 static void
 351 port_in_build_free(struct rte_swx_pipeline *p)
 352 {
 353         free(p->in);
 354         p->in = NULL;
 355 }
 356
 357 static void
 358 port_in_free(struct rte_swx_pipeline *p)
 359 {
 360         port_in_build_free(p);
 361
 362         /* Input ports. */
 363         for ( ; ; ) {
 364                 struct port_in *port;
 365
 366                 port = TAILQ_FIRST(&p->ports_in);
 367                 if (!port)
 368                         break;
 369
 370                 TAILQ_REMOVE(&p->ports_in, port, node);
 371                 port->type->ops.free(port->obj);
 372                 free(port);
 373         }
 374
 375         /* Input port types. */
 376         for ( ; ; ) {
 377                 struct port_in_type *elem;
 378
 379                 elem = TAILQ_FIRST(&p->port_in_types);
 380                 if (!elem)
 381                         break;
 382
 383                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 384                 free(elem);
 385         }
 386 }
 387
 388 /*
 389  * Output port.
 390  */
 391 static struct port_out_type *
 392 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 393 {
 394         struct port_out_type *elem;
 395
 396         if (!name)
 397                 return NULL;
 398
 399         TAILQ_FOREACH(elem, &p->port_out_types, node)
 400                 if (!strcmp(elem->name, name))
 401                         return elem;
 402
 403         return NULL;
 404 }
 405
 406 int
 407 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 408                                         const char *name,
 409                                         struct rte_swx_port_out_ops *ops)
 410 {
 411         struct port_out_type *elem;
 412
 413         CHECK(p, EINVAL);
 414         CHECK_NAME(name, EINVAL);
 415         CHECK(ops, EINVAL);
 416         CHECK(ops->create, EINVAL);
 417         CHECK(ops->free, EINVAL);
 418         CHECK(ops->pkt_tx, EINVAL);
 419         CHECK(ops->stats_read, EINVAL);
 420
 421         CHECK(!port_out_type_find(p, name), EEXIST);
 422
 423         /* Node allocation. */
 424         elem = calloc(1, sizeof(struct port_out_type));
 425         CHECK(elem, ENOMEM);
 426
 427         /* Node initialization. */
 428         strcpy(elem->name, name);
 429         memcpy(&elem->ops, ops, sizeof(*ops));
 430
 431         /* Node add to tailq. */
 432         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 433
 434         return 0;
 435 }
 436
 437 static struct port_out *
 438 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 439 {
 440         struct port_out *port;
 441
 442         TAILQ_FOREACH(port, &p->ports_out, node)
 443                 if (port->id == port_id)
 444                         return port;
 445
 446         return NULL;
 447 }
 448
 449 int
 450 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 451                                  uint32_t port_id,
 452                                  const char *port_type_name,
 453                                  void *args)
 454 {
 455         struct port_out_type *type = NULL;
 456         struct port_out *port = NULL;
 457         void *obj = NULL;
 458
 459         CHECK(p, EINVAL);
 460
 461         CHECK(!port_out_find(p, port_id), EINVAL);
 462
 463         CHECK_NAME(port_type_name, EINVAL);
 464         type = port_out_type_find(p, port_type_name);
 465         CHECK(type, EINVAL);
 466
 467         obj = type->ops.create(args);
 468         CHECK(obj, ENODEV);
 469
 470         /* Node allocation. */
 471         port = calloc(1, sizeof(struct port_out));
 472         CHECK(port, ENOMEM);
 473
 474         /* Node initialization. */
 475         port->type = type;
 476         port->obj = obj;
 477         port->id = port_id;
 478
 479         /* Node add to tailq. */
 480         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 481         if (p->n_ports_out < port_id + 1)
 482                 p->n_ports_out = port_id + 1;
 483
 484         return 0;
 485 }
 486
 487 static int
 488 port_out_build(struct rte_swx_pipeline *p)
 489 {
 490         struct port_out *port;
 491         uint32_t i;
 492
 493         CHECK(p->n_ports_out, EINVAL);
 494
 495         for (i = 0; i < p->n_ports_out; i++)
 496                 CHECK(port_out_find(p, i), EINVAL);
 497
 498         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 499         CHECK(p->out, ENOMEM);
 500
 501         TAILQ_FOREACH(port, &p->ports_out, node) {
 502                 struct port_out_runtime *out = &p->out[port->id];
 503
 504                 out->pkt_tx = port->type->ops.pkt_tx;
 505                 out->flush = port->type->ops.flush;
 506                 out->obj = port->obj;
 507         }
 508
 509         return 0;
 510 }
 511
 512 static void
 513 port_out_build_free(struct rte_swx_pipeline *p)
 514 {
 515         free(p->out);
 516         p->out = NULL;
 517 }
 518
 519 static void
 520 port_out_free(struct rte_swx_pipeline *p)
 521 {
 522         port_out_build_free(p);
 523
 524         /* Output ports. */
 525         for ( ; ; ) {
 526                 struct port_out *port;
 527
 528                 port = TAILQ_FIRST(&p->ports_out);
 529                 if (!port)
 530                         break;
 531
 532                 TAILQ_REMOVE(&p->ports_out, port, node);
 533                 port->type->ops.free(port->obj);
 534                 free(port);
 535         }
 536
 537         /* Output port types. */
 538         for ( ; ; ) {
 539                 struct port_out_type *elem;
 540
 541                 elem = TAILQ_FIRST(&p->port_out_types);
 542                 if (!elem)
 543                         break;
 544
 545                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 546                 free(elem);
 547         }
 548 }
 549
 550 /*
 551  * Extern object.
 552  */
 553 static struct extern_type *
 554 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 555 {
 556         struct extern_type *elem;
 557
 558         TAILQ_FOREACH(elem, &p->extern_types, node)
 559                 if (strcmp(elem->name, name) == 0)
 560                         return elem;
 561
 562         return NULL;
 563 }
 564
 565 static struct extern_type_member_func *
 566 extern_type_member_func_find(struct extern_type *type, const char *name)
 567 {
 568         struct extern_type_member_func *elem;
 569
 570         TAILQ_FOREACH(elem, &type->funcs, node)
 571                 if (strcmp(elem->name, name) == 0)
 572                         return elem;
 573
 574         return NULL;
 575 }
 576
 577 static struct extern_obj *
 578 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 579 {
 580         struct extern_obj *elem;
 581
 582         TAILQ_FOREACH(elem, &p->extern_objs, node)
 583                 if (strcmp(elem->name, name) == 0)
 584                         return elem;
 585
 586         return NULL;
 587 }
 588
 589 static struct extern_type_member_func *
 590 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 591                              const char *name,
 592                              struct extern_obj **obj)
 593 {
 594         struct extern_obj *object;
 595         struct extern_type_member_func *func;
 596         char *object_name, *func_name;
 597
 598         if (name[0] != 'e' || name[1] != '.')
 599                 return NULL;
 600
 601         object_name = strdup(&name[2]);
 602         if (!object_name)
 603                 return NULL;
 604
 605         func_name = strchr(object_name, '.');
 606         if (!func_name) {
 607                 free(object_name);
 608                 return NULL;
 609         }
 610
 611         *func_name = 0;
 612         func_name++;
 613
 614         object = extern_obj_find(p, object_name);
 615         if (!object) {
 616                 free(object_name);
 617                 return NULL;
 618         }
 619
 620         func = extern_type_member_func_find(object->type, func_name);
 621         if (!func) {
 622                 free(object_name);
 623                 return NULL;
 624         }
 625
 626         if (obj)
 627                 *obj = object;
 628
 629         free(object_name);
 630         return func;
 631 }
 632
 633 static struct field *
 634 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 635                                const char *name,
 636                                struct extern_obj **object)
 637 {
 638         struct extern_obj *obj;
 639         struct field *f;
 640         char *obj_name, *field_name;
 641
 642         if ((name[0] != 'e') || (name[1] != '.'))
 643                 return NULL;
 644
 645         obj_name = strdup(&name[2]);
 646         if (!obj_name)
 647                 return NULL;
 648
 649         field_name = strchr(obj_name, '.');
 650         if (!field_name) {
 651                 free(obj_name);
 652                 return NULL;
 653         }
 654
 655         *field_name = 0;
 656         field_name++;
 657
 658         obj = extern_obj_find(p, obj_name);
 659         if (!obj) {
 660                 free(obj_name);
 661                 return NULL;
 662         }
 663
 664         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 665         if (!f) {
 666                 free(obj_name);
 667                 return NULL;
 668         }
 669
 670         if (object)
 671                 *object = obj;
 672
 673         free(obj_name);
 674         return f;
 675 }
 676
 677 int
 678 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 679         const char *name,
 680         const char *mailbox_struct_type_name,
 681         rte_swx_extern_type_constructor_t constructor,
 682         rte_swx_extern_type_destructor_t destructor)
 683 {
 684         struct extern_type *elem;
 685         struct struct_type *mailbox_struct_type;
 686
 687         CHECK(p, EINVAL);
 688
 689         CHECK_NAME(name, EINVAL);
 690         CHECK(!extern_type_find(p, name), EEXIST);
 691
 692         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 693         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 694         CHECK(mailbox_struct_type, EINVAL);
 695         CHECK(!mailbox_struct_type->var_size, EINVAL);
 696
 697         CHECK(constructor, EINVAL);
 698         CHECK(destructor, EINVAL);
 699
 700         /* Node allocation. */
 701         elem = calloc(1, sizeof(struct extern_type));
 702         CHECK(elem, ENOMEM);
 703
 704         /* Node initialization. */
 705         strcpy(elem->name, name);
 706         elem->mailbox_struct_type = mailbox_struct_type;
 707         elem->constructor = constructor;
 708         elem->destructor = destructor;
 709         TAILQ_INIT(&elem->funcs);
 710
 711         /* Node add to tailq. */
 712         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 713
 714         return 0;
 715 }
 716
 717 int
 718 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 719         const char *extern_type_name,
 720         const char *name,
 721         rte_swx_extern_type_member_func_t member_func)
 722 {
 723         struct extern_type *type;
 724         struct extern_type_member_func *type_member;
 725
 726         CHECK(p, EINVAL);
 727
 728         CHECK_NAME(extern_type_name, EINVAL);
 729         type = extern_type_find(p, extern_type_name);
 730         CHECK(type, EINVAL);
 731         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 732
 733         CHECK_NAME(name, EINVAL);
 734         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 735
 736         CHECK(member_func, EINVAL);
 737
 738         /* Node allocation. */
 739         type_member = calloc(1, sizeof(struct extern_type_member_func));
 740         CHECK(type_member, ENOMEM);
 741
 742         /* Node initialization. */
 743         strcpy(type_member->name, name);
 744         type_member->func = member_func;
 745         type_member->id = type->n_funcs;
 746
 747         /* Node add to tailq. */
 748         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 749         type->n_funcs++;
 750
 751         return 0;
 752 }
 753
 754 int
 755 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 756                                       const char *extern_type_name,
 757                                       const char *name,
 758                                       const char *args)
 759 {
 760         struct extern_type *type;
 761         struct extern_obj *obj;
 762         void *obj_handle;
 763
 764         CHECK(p, EINVAL);
 765
 766         CHECK_NAME(extern_type_name, EINVAL);
 767         type = extern_type_find(p, extern_type_name);
 768         CHECK(type, EINVAL);
 769
 770         CHECK_NAME(name, EINVAL);
 771         CHECK(!extern_obj_find(p, name), EEXIST);
 772
 773         /* Node allocation. */
 774         obj = calloc(1, sizeof(struct extern_obj));
 775         CHECK(obj, ENOMEM);
 776
 777         /* Object construction. */
 778         obj_handle = type->constructor(args);
 779         if (!obj_handle) {
 780                 free(obj);
 781                 CHECK(0, ENODEV);
 782         }
 783
 784         /* Node initialization. */
 785         strcpy(obj->name, name);
 786         obj->type = type;
 787         obj->obj = obj_handle;
 788         obj->struct_id = p->n_structs;
 789         obj->id = p->n_extern_objs;
 790
 791         /* Node add to tailq. */
 792         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 793         p->n_extern_objs++;
 794         p->n_structs++;
 795
 796         return 0;
 797 }
 798
 799 static int
 800 extern_obj_build(struct rte_swx_pipeline *p)
 801 {
 802         uint32_t i;
 803
 804         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 805                 struct thread *t = &p->threads[i];
 806                 struct extern_obj *obj;
 807
 808                 t->extern_objs = calloc(p->n_extern_objs,
 809                                         sizeof(struct extern_obj_runtime));
 810                 CHECK(t->extern_objs, ENOMEM);
 811
 812                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 813                         struct extern_obj_runtime *r =
 814                                 &t->extern_objs[obj->id];
 815                         struct extern_type_member_func *func;
 816                         uint32_t mailbox_size =
 817                                 obj->type->mailbox_struct_type->n_bits / 8;
 818
 819                         r->obj = obj->obj;
 820
 821                         r->mailbox = calloc(1, mailbox_size);
 822                         CHECK(r->mailbox, ENOMEM);
 823
 824                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 825                                 r->funcs[func->id] = func->func;
 826
 827                         t->structs[obj->struct_id] = r->mailbox;
 828                 }
 829         }
 830
 831         return 0;
 832 }
 833
 834 static void
 835 extern_obj_build_free(struct rte_swx_pipeline *p)
 836 {
 837         uint32_t i;
 838
 839         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 840                 struct thread *t = &p->threads[i];
 841                 uint32_t j;
 842
 843                 if (!t->extern_objs)
 844                         continue;
 845
 846                 for (j = 0; j < p->n_extern_objs; j++) {
 847                         struct extern_obj_runtime *r = &t->extern_objs[j];
 848
 849                         free(r->mailbox);
 850                 }
 851
 852                 free(t->extern_objs);
 853                 t->extern_objs = NULL;
 854         }
 855 }
 856
 857 static void
 858 extern_obj_free(struct rte_swx_pipeline *p)
 859 {
 860         extern_obj_build_free(p);
 861
 862         /* Extern objects. */
 863         for ( ; ; ) {
 864                 struct extern_obj *elem;
 865
 866                 elem = TAILQ_FIRST(&p->extern_objs);
 867                 if (!elem)
 868                         break;
 869
 870                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 871                 if (elem->obj)
 872                         elem->type->destructor(elem->obj);
 873                 free(elem);
 874         }
 875
 876         /* Extern types. */
 877         for ( ; ; ) {
 878                 struct extern_type *elem;
 879
 880                 elem = TAILQ_FIRST(&p->extern_types);
 881                 if (!elem)
 882                         break;
 883
 884                 TAILQ_REMOVE(&p->extern_types, elem, node);
 885
 886                 for ( ; ; ) {
 887                         struct extern_type_member_func *func;
 888
 889                         func = TAILQ_FIRST(&elem->funcs);
 890                         if (!func)
 891                                 break;
 892
 893                         TAILQ_REMOVE(&elem->funcs, func, node);
 894                         free(func);
 895                 }
 896
 897                 free(elem);
 898         }
 899 }
 900
 901 /*
 902  * Extern function.
 903  */
 904 static struct extern_func *
 905 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 906 {
 907         struct extern_func *elem;
 908
 909         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 910                 if (strcmp(elem->name, name) == 0)
 911                         return elem;
 912
 913         return NULL;
 914 }
 915
 916 static struct extern_func *
 917 extern_func_parse(struct rte_swx_pipeline *p,
 918                   const char *name)
 919 {
 920         if (name[0] != 'f' || name[1] != '.')
 921                 return NULL;
 922
 923         return extern_func_find(p, &name[2]);
 924 }
 925
 926 static struct field *
 927 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 928                                 const char *name,
 929                                 struct extern_func **function)
 930 {
 931         struct extern_func *func;
 932         struct field *f;
 933         char *func_name, *field_name;
 934
 935         if ((name[0] != 'f') || (name[1] != '.'))
 936                 return NULL;
 937
 938         func_name = strdup(&name[2]);
 939         if (!func_name)
 940                 return NULL;
 941
 942         field_name = strchr(func_name, '.');
 943         if (!field_name) {
 944                 free(func_name);
 945                 return NULL;
 946         }
 947
 948         *field_name = 0;
 949         field_name++;
 950
 951         func = extern_func_find(p, func_name);
 952         if (!func) {
 953                 free(func_name);
 954                 return NULL;
 955         }
 956
 957         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 958         if (!f) {
 959                 free(func_name);
 960                 return NULL;
 961         }
 962
 963         if (function)
 964                 *function = func;
 965
 966         free(func_name);
 967         return f;
 968 }
 969
 970 int
 971 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 972                                       const char *name,
 973                                       const char *mailbox_struct_type_name,
 974                                       rte_swx_extern_func_t func)
 975 {
 976         struct extern_func *f;
 977         struct struct_type *mailbox_struct_type;
 978
 979         CHECK(p, EINVAL);
 980
 981         CHECK_NAME(name, EINVAL);
 982         CHECK(!extern_func_find(p, name), EEXIST);
 983
 984         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 985         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 986         CHECK(mailbox_struct_type, EINVAL);
 987         CHECK(!mailbox_struct_type->var_size, EINVAL);
 988
 989         CHECK(func, EINVAL);
 990
 991         /* Node allocation. */
 992         f = calloc(1, sizeof(struct extern_func));
 993         CHECK(func, ENOMEM);
 994
 995         /* Node initialization. */
 996         strcpy(f->name, name);
 997         f->mailbox_struct_type = mailbox_struct_type;
 998         f->func = func;
 999         f->struct_id = p->n_structs;
1000         f->id = p->n_extern_funcs;
1001
1002         /* Node add to tailq. */
1003         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1004         p->n_extern_funcs++;
1005         p->n_structs++;
1006
1007         return 0;
1008 }
1009
1010 static int
1011 extern_func_build(struct rte_swx_pipeline *p)
1012 {
1013         uint32_t i;
1014
1015         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1016                 struct thread *t = &p->threads[i];
1017                 struct extern_func *func;
1018
1019                 /* Memory allocation. */
1020                 t->extern_funcs = calloc(p->n_extern_funcs,
1021                                          sizeof(struct extern_func_runtime));
1022                 CHECK(t->extern_funcs, ENOMEM);
1023
1024                 /* Extern function. */
1025                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1026                         struct extern_func_runtime *r =
1027                                 &t->extern_funcs[func->id];
1028                         uint32_t mailbox_size =
1029                                 func->mailbox_struct_type->n_bits / 8;
1030
1031                         r->func = func->func;
1032
1033                         r->mailbox = calloc(1, mailbox_size);
1034                         CHECK(r->mailbox, ENOMEM);
1035
1036                         t->structs[func->struct_id] = r->mailbox;
1037                 }
1038         }
1039
1040         return 0;
1041 }
1042
1043 static void
1044 extern_func_build_free(struct rte_swx_pipeline *p)
1045 {
1046         uint32_t i;
1047
1048         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1049                 struct thread *t = &p->threads[i];
1050                 uint32_t j;
1051
1052                 if (!t->extern_funcs)
1053                         continue;
1054
1055                 for (j = 0; j < p->n_extern_funcs; j++) {
1056                         struct extern_func_runtime *r = &t->extern_funcs[j];
1057
1058                         free(r->mailbox);
1059                 }
1060
1061                 free(t->extern_funcs);
1062                 t->extern_funcs = NULL;
1063         }
1064 }
1065
1066 static void
1067 extern_func_free(struct rte_swx_pipeline *p)
1068 {
1069         extern_func_build_free(p);
1070
1071         for ( ; ; ) {
1072                 struct extern_func *elem;
1073
1074                 elem = TAILQ_FIRST(&p->extern_funcs);
1075                 if (!elem)
1076                         break;
1077
1078                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1079                 free(elem);
1080         }
1081 }
1082
1083 /*
1084  * Header.
1085  */
1086 static struct header *
1087 header_find(struct rte_swx_pipeline *p, const char *name)
1088 {
1089         struct header *elem;
1090
1091         TAILQ_FOREACH(elem, &p->headers, node)
1092                 if (strcmp(elem->name, name) == 0)
1093                         return elem;
1094
1095         return NULL;
1096 }
1097
1098 static struct header *
1099 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1100 {
1101         struct header *elem;
1102
1103         TAILQ_FOREACH(elem, &p->headers, node)
1104                 if (elem->struct_id == struct_id)
1105                         return elem;
1106
1107         return NULL;
1108 }
1109
1110 static struct header *
1111 header_parse(struct rte_swx_pipeline *p,
1112              const char *name)
1113 {
1114         if (name[0] != 'h' || name[1] != '.')
1115                 return NULL;
1116
1117         return header_find(p, &name[2]);
1118 }
1119
1120 static struct field *
1121 header_field_parse(struct rte_swx_pipeline *p,
1122                    const char *name,
1123                    struct header **header)
1124 {
1125         struct header *h;
1126         struct field *f;
1127         char *header_name, *field_name;
1128
1129         if ((name[0] != 'h') || (name[1] != '.'))
1130                 return NULL;
1131
1132         header_name = strdup(&name[2]);
1133         if (!header_name)
1134                 return NULL;
1135
1136         field_name = strchr(header_name, '.');
1137         if (!field_name) {
1138                 free(header_name);
1139                 return NULL;
1140         }
1141
1142         *field_name = 0;
1143         field_name++;
1144
1145         h = header_find(p, header_name);
1146         if (!h) {
1147                 free(header_name);
1148                 return NULL;
1149         }
1150
1151         f = struct_type_field_find(h->st, field_name);
1152         if (!f) {
1153                 free(header_name);
1154                 return NULL;
1155         }
1156
1157         if (header)
1158                 *header = h;
1159
1160         free(header_name);
1161         return f;
1162 }
1163
1164 int
1165 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1166                                         const char *name,
1167                                         const char *struct_type_name)
1168 {
1169         struct struct_type *st;
1170         struct header *h;
1171         size_t n_headers_max;
1172
1173         CHECK(p, EINVAL);
1174         CHECK_NAME(name, EINVAL);
1175         CHECK_NAME(struct_type_name, EINVAL);
1176
1177         CHECK(!header_find(p, name), EEXIST);
1178
1179         st = struct_type_find(p, struct_type_name);
1180         CHECK(st, EINVAL);
1181
1182         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1183         CHECK(p->n_headers < n_headers_max, ENOSPC);
1184
1185         /* Node allocation. */
1186         h = calloc(1, sizeof(struct header));
1187         CHECK(h, ENOMEM);
1188
1189         /* Node initialization. */
1190         strcpy(h->name, name);
1191         h->st = st;
1192         h->struct_id = p->n_structs;
1193         h->id = p->n_headers;
1194
1195         /* Node add to tailq. */
1196         TAILQ_INSERT_TAIL(&p->headers, h, node);
1197         p->n_headers++;
1198         p->n_structs++;
1199
1200         return 0;
1201 }
1202
1203 static int
1204 header_build(struct rte_swx_pipeline *p)
1205 {
1206         struct header *h;
1207         uint32_t n_bytes = 0, i;
1208
1209         TAILQ_FOREACH(h, &p->headers, node) {
1210                 n_bytes += h->st->n_bits / 8;
1211         }
1212
1213         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1214                 struct thread *t = &p->threads[i];
1215                 uint32_t offset = 0;
1216
1217                 t->headers = calloc(p->n_headers,
1218                                     sizeof(struct header_runtime));
1219                 CHECK(t->headers, ENOMEM);
1220
1221                 t->headers_out = calloc(p->n_headers,
1222                                         sizeof(struct header_out_runtime));
1223                 CHECK(t->headers_out, ENOMEM);
1224
1225                 t->header_storage = calloc(1, n_bytes);
1226                 CHECK(t->header_storage, ENOMEM);
1227
1228                 t->header_out_storage = calloc(1, n_bytes);
1229                 CHECK(t->header_out_storage, ENOMEM);
1230
1231                 TAILQ_FOREACH(h, &p->headers, node) {
1232                         uint8_t *header_storage;
1233                         uint32_t n_bytes =  h->st->n_bits / 8;
1234
1235                         header_storage = &t->header_storage[offset];
1236                         offset += n_bytes;
1237
1238                         t->headers[h->id].ptr0 = header_storage;
1239                         t->headers[h->id].n_bytes = n_bytes;
1240
1241                         t->structs[h->struct_id] = header_storage;
1242                 }
1243         }
1244
1245         return 0;
1246 }
1247
1248 static void
1249 header_build_free(struct rte_swx_pipeline *p)
1250 {
1251         uint32_t i;
1252
1253         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1254                 struct thread *t = &p->threads[i];
1255
1256                 free(t->headers_out);
1257                 t->headers_out = NULL;
1258
1259                 free(t->headers);
1260                 t->headers = NULL;
1261
1262                 free(t->header_out_storage);
1263                 t->header_out_storage = NULL;
1264
1265                 free(t->header_storage);
1266                 t->header_storage = NULL;
1267         }
1268 }
1269
1270 static void
1271 header_free(struct rte_swx_pipeline *p)
1272 {
1273         header_build_free(p);
1274
1275         for ( ; ; ) {
1276                 struct header *elem;
1277
1278                 elem = TAILQ_FIRST(&p->headers);
1279                 if (!elem)
1280                         break;
1281
1282                 TAILQ_REMOVE(&p->headers, elem, node);
1283                 free(elem);
1284         }
1285 }
1286
1287 /*
1288  * Meta-data.
1289  */
1290 static struct field *
1291 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1292 {
1293         if (!p->metadata_st)
1294                 return NULL;
1295
1296         if (name[0] != 'm' || name[1] != '.')
1297                 return NULL;
1298
1299         return struct_type_field_find(p->metadata_st, &name[2]);
1300 }
1301
1302 int
1303 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1304                                           const char *struct_type_name)
1305 {
1306         struct struct_type *st = NULL;
1307
1308         CHECK(p, EINVAL);
1309
1310         CHECK_NAME(struct_type_name, EINVAL);
1311         st  = struct_type_find(p, struct_type_name);
1312         CHECK(st, EINVAL);
1313         CHECK(!st->var_size, EINVAL);
1314         CHECK(!p->metadata_st, EINVAL);
1315
1316         p->metadata_st = st;
1317         p->metadata_struct_id = p->n_structs;
1318
1319         p->n_structs++;
1320
1321         return 0;
1322 }
1323
1324 static int
1325 metadata_build(struct rte_swx_pipeline *p)
1326 {
1327         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1328         uint32_t i;
1329
1330         /* Thread-level initialization. */
1331         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1332                 struct thread *t = &p->threads[i];
1333                 uint8_t *metadata;
1334
1335                 metadata = calloc(1, n_bytes);
1336                 CHECK(metadata, ENOMEM);
1337
1338                 t->metadata = metadata;
1339                 t->structs[p->metadata_struct_id] = metadata;
1340         }
1341
1342         return 0;
1343 }
1344
1345 static void
1346 metadata_build_free(struct rte_swx_pipeline *p)
1347 {
1348         uint32_t i;
1349
1350         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1351                 struct thread *t = &p->threads[i];
1352
1353                 free(t->metadata);
1354                 t->metadata = NULL;
1355         }
1356 }
1357
1358 static void
1359 metadata_free(struct rte_swx_pipeline *p)
1360 {
1361         metadata_build_free(p);
1362 }
1363
1364 /*
1365  * Instruction.
1366  */
1367 static int
1368 instruction_is_tx(enum instruction_type type)
1369 {
1370         switch (type) {
1371         case INSTR_TX:
1372         case INSTR_TX_I:
1373                 return 1;
1374
1375         default:
1376                 return 0;
1377         }
1378 }
1379
1380 static int
1381 instruction_does_tx(struct instruction *instr)
1382 {
1383         switch (instr->type) {
1384         case INSTR_TX:
1385         case INSTR_TX_I:
1386         case INSTR_HDR_EMIT_TX:
1387         case INSTR_HDR_EMIT2_TX:
1388         case INSTR_HDR_EMIT3_TX:
1389         case INSTR_HDR_EMIT4_TX:
1390         case INSTR_HDR_EMIT5_TX:
1391         case INSTR_HDR_EMIT6_TX:
1392         case INSTR_HDR_EMIT7_TX:
1393         case INSTR_HDR_EMIT8_TX:
1394                 return 1;
1395         default:
1396                 return 0;
1397         }
1398 }
1399
1400 static int
1401 instruction_is_jmp(struct instruction *instr)
1402 {
1403         switch (instr->type) {
1404         case INSTR_JMP:
1405         case INSTR_JMP_VALID:
1406         case INSTR_JMP_INVALID:
1407         case INSTR_JMP_HIT:
1408         case INSTR_JMP_MISS:
1409         case INSTR_JMP_ACTION_HIT:
1410         case INSTR_JMP_ACTION_MISS:
1411         case INSTR_JMP_EQ:
1412         case INSTR_JMP_EQ_MH:
1413         case INSTR_JMP_EQ_HM:
1414         case INSTR_JMP_EQ_HH:
1415         case INSTR_JMP_EQ_I:
1416         case INSTR_JMP_NEQ:
1417         case INSTR_JMP_NEQ_MH:
1418         case INSTR_JMP_NEQ_HM:
1419         case INSTR_JMP_NEQ_HH:
1420         case INSTR_JMP_NEQ_I:
1421         case INSTR_JMP_LT:
1422         case INSTR_JMP_LT_MH:
1423         case INSTR_JMP_LT_HM:
1424         case INSTR_JMP_LT_HH:
1425         case INSTR_JMP_LT_MI:
1426         case INSTR_JMP_LT_HI:
1427         case INSTR_JMP_GT:
1428         case INSTR_JMP_GT_MH:
1429         case INSTR_JMP_GT_HM:
1430         case INSTR_JMP_GT_HH:
1431         case INSTR_JMP_GT_MI:
1432         case INSTR_JMP_GT_HI:
1433                 return 1;
1434
1435         default:
1436                 return 0;
1437         }
1438 }
1439
1440 static int
1441 instruction_does_thread_yield(struct instruction *instr)
1442 {
1443         switch (instr->type) {
1444         case INSTR_RX:
1445         case INSTR_TABLE:
1446         case INSTR_TABLE_AF:
1447         case INSTR_SELECTOR:
1448         case INSTR_LEARNER:
1449         case INSTR_LEARNER_AF:
1450         case INSTR_EXTERN_OBJ:
1451         case INSTR_EXTERN_FUNC:
1452                 return 1;
1453         default:
1454                 return 0;
1455         }
1456 }
1457
1458 static struct field *
1459 action_field_parse(struct action *action, const char *name);
1460
1461 static struct field *
1462 struct_field_parse(struct rte_swx_pipeline *p,
1463                    struct action *action,
1464                    const char *name,
1465                    uint32_t *struct_id)
1466 {
1467         struct field *f;
1468
1469         switch (name[0]) {
1470         case 'h':
1471         {
1472                 struct header *header;
1473
1474                 f = header_field_parse(p, name, &header);
1475                 if (!f)
1476                         return NULL;
1477
1478                 *struct_id = header->struct_id;
1479                 return f;
1480         }
1481
1482         case 'm':
1483         {
1484                 f = metadata_field_parse(p, name);
1485                 if (!f)
1486                         return NULL;
1487
1488                 *struct_id = p->metadata_struct_id;
1489                 return f;
1490         }
1491
1492         case 't':
1493         {
1494                 if (!action)
1495                         return NULL;
1496
1497                 f = action_field_parse(action, name);
1498                 if (!f)
1499                         return NULL;
1500
1501                 *struct_id = 0;
1502                 return f;
1503         }
1504
1505         case 'e':
1506         {
1507                 struct extern_obj *obj;
1508
1509                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1510                 if (!f)
1511                         return NULL;
1512
1513                 *struct_id = obj->struct_id;
1514                 return f;
1515         }
1516
1517         case 'f':
1518         {
1519                 struct extern_func *func;
1520
1521                 f = extern_func_mailbox_field_parse(p, name, &func);
1522                 if (!f)
1523                         return NULL;
1524
1525                 *struct_id = func->struct_id;
1526                 return f;
1527         }
1528
1529         default:
1530                 return NULL;
1531         }
1532 }
1533
1534 /*
1535  * rx.
1536  */
1537 static int
1538 instr_rx_translate(struct rte_swx_pipeline *p,
1539                    struct action *action,
1540                    char **tokens,
1541                    int n_tokens,
1542                    struct instruction *instr,
1543                    struct instruction_data *data __rte_unused)
1544 {
1545         struct field *f;
1546
1547         CHECK(!action, EINVAL);
1548         CHECK(n_tokens == 2, EINVAL);
1549
1550         f = metadata_field_parse(p, tokens[1]);
1551         CHECK(f, EINVAL);
1552
1553         instr->type = INSTR_RX;
1554         instr->io.io.offset = f->offset / 8;
1555         instr->io.io.n_bits = f->n_bits;
1556         return 0;
1557 }
1558
1559 /*
1560  * tx.
1561  */
1562 static int
1563 instr_tx_translate(struct rte_swx_pipeline *p,
1564                    struct action *action __rte_unused,
1565                    char **tokens,
1566                    int n_tokens,
1567                    struct instruction *instr,
1568                    struct instruction_data *data __rte_unused)
1569 {
1570         char *port = tokens[1];
1571         struct field *f;
1572         uint32_t port_val;
1573
1574         CHECK(n_tokens == 2, EINVAL);
1575
1576         f = metadata_field_parse(p, port);
1577         if (f) {
1578                 instr->type = INSTR_TX;
1579                 instr->io.io.offset = f->offset / 8;
1580                 instr->io.io.n_bits = f->n_bits;
1581                 return 0;
1582         }
1583
1584         /* TX_I. */
1585         port_val = strtoul(port, &port, 0);
1586         CHECK(!port[0], EINVAL);
1587
1588         instr->type = INSTR_TX_I;
1589         instr->io.io.val = port_val;
1590         return 0;
1591 }
1592
1593 static int
1594 instr_drop_translate(struct rte_swx_pipeline *p,
1595                      struct action *action __rte_unused,
1596                      char **tokens __rte_unused,
1597                      int n_tokens,
1598                      struct instruction *instr,
1599                      struct instruction_data *data __rte_unused)
1600 {
1601         CHECK(n_tokens == 1, EINVAL);
1602
1603         /* TX_I. */
1604         instr->type = INSTR_TX_I;
1605         instr->io.io.val = p->n_ports_out - 1;
1606         return 0;
1607 }
1608
1609 static inline void
1610 instr_tx_exec(struct rte_swx_pipeline *p)
1611 {
1612         struct thread *t = &p->threads[p->thread_id];
1613         struct instruction *ip = t->ip;
1614
1615         __instr_tx_exec(p, t, ip);
1616
1617         /* Thread. */
1618         thread_ip_reset(p, t);
1619         instr_rx_exec(p);
1620 }
1621
1622 static inline void
1623 instr_tx_i_exec(struct rte_swx_pipeline *p)
1624 {
1625         struct thread *t = &p->threads[p->thread_id];
1626         struct instruction *ip = t->ip;
1627
1628         __instr_tx_i_exec(p, t, ip);
1629
1630         /* Thread. */
1631         thread_ip_reset(p, t);
1632         instr_rx_exec(p);
1633 }
1634
1635 /*
1636  * extract.
1637  */
1638 static int
1639 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1640                             struct action *action,
1641                             char **tokens,
1642                             int n_tokens,
1643                             struct instruction *instr,
1644                             struct instruction_data *data __rte_unused)
1645 {
1646         struct header *h;
1647
1648         CHECK(!action, EINVAL);
1649         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1650
1651         h = header_parse(p, tokens[1]);
1652         CHECK(h, EINVAL);
1653
1654         if (n_tokens == 2) {
1655                 CHECK(!h->st->var_size, EINVAL);
1656
1657                 instr->type = INSTR_HDR_EXTRACT;
1658                 instr->io.hdr.header_id[0] = h->id;
1659                 instr->io.hdr.struct_id[0] = h->struct_id;
1660                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1661         } else {
1662                 struct field *mf;
1663
1664                 CHECK(h->st->var_size, EINVAL);
1665
1666                 mf = metadata_field_parse(p, tokens[2]);
1667                 CHECK(mf, EINVAL);
1668                 CHECK(!mf->var_size, EINVAL);
1669
1670                 instr->type = INSTR_HDR_EXTRACT_M;
1671                 instr->io.io.offset = mf->offset / 8;
1672                 instr->io.io.n_bits = mf->n_bits;
1673                 instr->io.hdr.header_id[0] = h->id;
1674                 instr->io.hdr.struct_id[0] = h->struct_id;
1675                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1676         }
1677
1678         return 0;
1679 }
1680
1681 static int
1682 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1683                               struct action *action,
1684                               char **tokens,
1685                               int n_tokens,
1686                               struct instruction *instr,
1687                               struct instruction_data *data __rte_unused)
1688 {
1689         struct header *h;
1690
1691         CHECK(!action, EINVAL);
1692         CHECK(n_tokens == 2, EINVAL);
1693
1694         h = header_parse(p, tokens[1]);
1695         CHECK(h, EINVAL);
1696         CHECK(!h->st->var_size, EINVAL);
1697
1698         instr->type = INSTR_HDR_LOOKAHEAD;
1699         instr->io.hdr.header_id[0] = h->id;
1700         instr->io.hdr.struct_id[0] = h->struct_id;
1701         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1702
1703         return 0;
1704 }
1705
1706 static inline void
1707 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1708 {
1709         struct thread *t = &p->threads[p->thread_id];
1710         struct instruction *ip = t->ip;
1711
1712         __instr_hdr_extract_exec(p, t, ip);
1713
1714         /* Thread. */
1715         thread_ip_inc(p);
1716 }
1717
1718 static inline void
1719 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1720 {
1721         struct thread *t = &p->threads[p->thread_id];
1722         struct instruction *ip = t->ip;
1723
1724         __instr_hdr_extract2_exec(p, t, ip);
1725
1726         /* Thread. */
1727         thread_ip_inc(p);
1728 }
1729
1730 static inline void
1731 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1732 {
1733         struct thread *t = &p->threads[p->thread_id];
1734         struct instruction *ip = t->ip;
1735
1736         __instr_hdr_extract3_exec(p, t, ip);
1737
1738         /* Thread. */
1739         thread_ip_inc(p);
1740 }
1741
1742 static inline void
1743 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1744 {
1745         struct thread *t = &p->threads[p->thread_id];
1746         struct instruction *ip = t->ip;
1747
1748         __instr_hdr_extract4_exec(p, t, ip);
1749
1750         /* Thread. */
1751         thread_ip_inc(p);
1752 }
1753
1754 static inline void
1755 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1756 {
1757         struct thread *t = &p->threads[p->thread_id];
1758         struct instruction *ip = t->ip;
1759
1760         __instr_hdr_extract5_exec(p, t, ip);
1761
1762         /* Thread. */
1763         thread_ip_inc(p);
1764 }
1765
1766 static inline void
1767 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1768 {
1769         struct thread *t = &p->threads[p->thread_id];
1770         struct instruction *ip = t->ip;
1771
1772         __instr_hdr_extract6_exec(p, t, ip);
1773
1774         /* Thread. */
1775         thread_ip_inc(p);
1776 }
1777
1778 static inline void
1779 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1780 {
1781         struct thread *t = &p->threads[p->thread_id];
1782         struct instruction *ip = t->ip;
1783
1784         __instr_hdr_extract7_exec(p, t, ip);
1785
1786         /* Thread. */
1787         thread_ip_inc(p);
1788 }
1789
1790 static inline void
1791 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1792 {
1793         struct thread *t = &p->threads[p->thread_id];
1794         struct instruction *ip = t->ip;
1795
1796         __instr_hdr_extract8_exec(p, t, ip);
1797
1798         /* Thread. */
1799         thread_ip_inc(p);
1800 }
1801
1802 static inline void
1803 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1804 {
1805         struct thread *t = &p->threads[p->thread_id];
1806         struct instruction *ip = t->ip;
1807
1808         __instr_hdr_extract_m_exec(p, t, ip);
1809
1810         /* Thread. */
1811         thread_ip_inc(p);
1812 }
1813
1814 static inline void
1815 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1816 {
1817         struct thread *t = &p->threads[p->thread_id];
1818         struct instruction *ip = t->ip;
1819
1820         __instr_hdr_lookahead_exec(p, t, ip);
1821
1822         /* Thread. */
1823         thread_ip_inc(p);
1824 }
1825
1826 /*
1827  * emit.
1828  */
1829 static int
1830 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1831                          struct action *action __rte_unused,
1832                          char **tokens,
1833                          int n_tokens,
1834                          struct instruction *instr,
1835                          struct instruction_data *data __rte_unused)
1836 {
1837         struct header *h;
1838
1839         CHECK(n_tokens == 2, EINVAL);
1840
1841         h = header_parse(p, tokens[1]);
1842         CHECK(h, EINVAL);
1843
1844         instr->type = INSTR_HDR_EMIT;
1845         instr->io.hdr.header_id[0] = h->id;
1846         instr->io.hdr.struct_id[0] = h->struct_id;
1847         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1848         return 0;
1849 }
1850
1851 static inline void
1852 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1853 {
1854         struct thread *t = &p->threads[p->thread_id];
1855         struct instruction *ip = t->ip;
1856
1857         __instr_hdr_emit_exec(p, t, ip);
1858
1859         /* Thread. */
1860         thread_ip_inc(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit2_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit3_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit4_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit5_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 static inline void
1929 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1930 {
1931         struct thread *t = &p->threads[p->thread_id];
1932         struct instruction *ip = t->ip;
1933
1934         __instr_hdr_emit6_tx_exec(p, t, ip);
1935
1936         /* Thread. */
1937         thread_ip_reset(p, t);
1938         instr_rx_exec(p);
1939 }
1940
1941 static inline void
1942 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1943 {
1944         struct thread *t = &p->threads[p->thread_id];
1945         struct instruction *ip = t->ip;
1946
1947         __instr_hdr_emit7_tx_exec(p, t, ip);
1948
1949         /* Thread. */
1950         thread_ip_reset(p, t);
1951         instr_rx_exec(p);
1952 }
1953
1954 static inline void
1955 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1956 {
1957         struct thread *t = &p->threads[p->thread_id];
1958         struct instruction *ip = t->ip;
1959
1960         __instr_hdr_emit8_tx_exec(p, t, ip);
1961
1962         /* Thread. */
1963         thread_ip_reset(p, t);
1964         instr_rx_exec(p);
1965 }
1966
1967 /*
1968  * validate.
1969  */
1970 static int
1971 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1972                              struct action *action __rte_unused,
1973                              char **tokens,
1974                              int n_tokens,
1975                              struct instruction *instr,
1976                              struct instruction_data *data __rte_unused)
1977 {
1978         struct header *h;
1979
1980         CHECK(n_tokens == 2, EINVAL);
1981
1982         h = header_parse(p, tokens[1]);
1983         CHECK(h, EINVAL);
1984
1985         instr->type = INSTR_HDR_VALIDATE;
1986         instr->valid.header_id = h->id;
1987         return 0;
1988 }
1989
1990 static inline void
1991 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1992 {
1993         struct thread *t = &p->threads[p->thread_id];
1994         struct instruction *ip = t->ip;
1995
1996         __instr_hdr_validate_exec(p, t, ip);
1997
1998         /* Thread. */
1999         thread_ip_inc(p);
2000 }
2001
2002 /*
2003  * invalidate.
2004  */
2005 static int
2006 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2007                                struct action *action __rte_unused,
2008                                char **tokens,
2009                                int n_tokens,
2010                                struct instruction *instr,
2011                                struct instruction_data *data __rte_unused)
2012 {
2013         struct header *h;
2014
2015         CHECK(n_tokens == 2, EINVAL);
2016
2017         h = header_parse(p, tokens[1]);
2018         CHECK(h, EINVAL);
2019
2020         instr->type = INSTR_HDR_INVALIDATE;
2021         instr->valid.header_id = h->id;
2022         return 0;
2023 }
2024
2025 static inline void
2026 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2027 {
2028         struct thread *t = &p->threads[p->thread_id];
2029         struct instruction *ip = t->ip;
2030
2031         __instr_hdr_invalidate_exec(p, t, ip);
2032
2033         /* Thread. */
2034         thread_ip_inc(p);
2035 }
2036
2037 /*
2038  * table.
2039  */
2040 static struct table *
2041 table_find(struct rte_swx_pipeline *p, const char *name);
2042
2043 static struct selector *
2044 selector_find(struct rte_swx_pipeline *p, const char *name);
2045
2046 static struct learner *
2047 learner_find(struct rte_swx_pipeline *p, const char *name);
2048
2049 static int
2050 instr_table_translate(struct rte_swx_pipeline *p,
2051                       struct action *action,
2052                       char **tokens,
2053                       int n_tokens,
2054                       struct instruction *instr,
2055                       struct instruction_data *data __rte_unused)
2056 {
2057         struct table *t;
2058         struct selector *s;
2059         struct learner *l;
2060
2061         CHECK(!action, EINVAL);
2062         CHECK(n_tokens == 2, EINVAL);
2063
2064         t = table_find(p, tokens[1]);
2065         if (t) {
2066                 instr->type = INSTR_TABLE;
2067                 instr->table.table_id = t->id;
2068                 return 0;
2069         }
2070
2071         s = selector_find(p, tokens[1]);
2072         if (s) {
2073                 instr->type = INSTR_SELECTOR;
2074                 instr->table.table_id = s->id;
2075                 return 0;
2076         }
2077
2078         l = learner_find(p, tokens[1]);
2079         if (l) {
2080                 instr->type = INSTR_LEARNER;
2081                 instr->table.table_id = l->id;
2082                 return 0;
2083         }
2084
2085         CHECK(0, EINVAL);
2086 }
2087
2088 static inline void
2089 instr_table_exec(struct rte_swx_pipeline *p)
2090 {
2091         struct thread *t = &p->threads[p->thread_id];
2092         struct instruction *ip = t->ip;
2093         uint32_t table_id = ip->table.table_id;
2094         struct rte_swx_table_state *ts = &t->table_state[table_id];
2095         struct table_runtime *table = &t->tables[table_id];
2096         struct table_statistics *stats = &p->table_stats[table_id];
2097         uint64_t action_id, n_pkts_hit, n_pkts_action;
2098         uint8_t *action_data;
2099         int done, hit;
2100
2101         /* Table. */
2102         done = table->func(ts->obj,
2103                            table->mailbox,
2104                            table->key,
2105                            &action_id,
2106                            &action_data,
2107                            &hit);
2108         if (!done) {
2109                 /* Thread. */
2110                 TRACE("[Thread %2u] table %u (not finalized)\n",
2111                       p->thread_id,
2112                       table_id);
2113
2114                 thread_yield(p);
2115                 return;
2116         }
2117
2118         action_id = hit ? action_id : ts->default_action_id;
2119         action_data = hit ? action_data : ts->default_action_data;
2120         n_pkts_hit = stats->n_pkts_hit[hit];
2121         n_pkts_action = stats->n_pkts_action[action_id];
2122
2123         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2124               p->thread_id,
2125               table_id,
2126               hit ? "hit" : "miss",
2127               (uint32_t)action_id);
2128
2129         t->action_id = action_id;
2130         t->structs[0] = action_data;
2131         t->hit = hit;
2132         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2133         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2134
2135         /* Thread. */
2136         thread_ip_action_call(p, t, action_id);
2137 }
2138
2139 static inline void
2140 instr_table_af_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t table_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[table_id];
2146         struct table_runtime *table = &t->tables[table_id];
2147         struct table_statistics *stats = &p->table_stats[table_id];
2148         uint64_t action_id, n_pkts_hit, n_pkts_action;
2149         uint8_t *action_data;
2150         action_func_t action_func;
2151         int done, hit;
2152
2153         /* Table. */
2154         done = table->func(ts->obj,
2155                            table->mailbox,
2156                            table->key,
2157                            &action_id,
2158                            &action_data,
2159                            &hit);
2160         if (!done) {
2161                 /* Thread. */
2162                 TRACE("[Thread %2u] table %u (not finalized)\n",
2163                       p->thread_id,
2164                       table_id);
2165
2166                 thread_yield(p);
2167                 return;
2168         }
2169
2170         action_id = hit ? action_id : ts->default_action_id;
2171         action_data = hit ? action_data : ts->default_action_data;
2172         action_func = p->action_funcs[action_id];
2173         n_pkts_hit = stats->n_pkts_hit[hit];
2174         n_pkts_action = stats->n_pkts_action[action_id];
2175
2176         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2177               p->thread_id,
2178               table_id,
2179               hit ? "hit" : "miss",
2180               (uint32_t)action_id);
2181
2182         t->action_id = action_id;
2183         t->structs[0] = action_data;
2184         t->hit = hit;
2185         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2186         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2187
2188         /* Thread. */
2189         thread_ip_inc(p);
2190
2191         /* Action. */
2192         action_func(p);
2193 }
2194
2195 static inline void
2196 instr_selector_exec(struct rte_swx_pipeline *p)
2197 {
2198         struct thread *t = &p->threads[p->thread_id];
2199         struct instruction *ip = t->ip;
2200         uint32_t selector_id = ip->table.table_id;
2201         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2202         struct selector_runtime *selector = &t->selectors[selector_id];
2203         struct selector_statistics *stats = &p->selector_stats[selector_id];
2204         uint64_t n_pkts = stats->n_pkts;
2205         int done;
2206
2207         /* Table. */
2208         done = rte_swx_table_selector_select(ts->obj,
2209                            selector->mailbox,
2210                            selector->group_id_buffer,
2211                            selector->selector_buffer,
2212                            selector->member_id_buffer);
2213         if (!done) {
2214                 /* Thread. */
2215                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2216                       p->thread_id,
2217                       selector_id);
2218
2219                 thread_yield(p);
2220                 return;
2221         }
2222
2223
2224         TRACE("[Thread %2u] selector %u\n",
2225               p->thread_id,
2226               selector_id);
2227
2228         stats->n_pkts = n_pkts + 1;
2229
2230         /* Thread. */
2231         thread_ip_inc(p);
2232 }
2233
2234 static inline void
2235 instr_learner_exec(struct rte_swx_pipeline *p)
2236 {
2237         struct thread *t = &p->threads[p->thread_id];
2238         struct instruction *ip = t->ip;
2239         uint32_t learner_id = ip->table.table_id;
2240         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2241                 p->n_selectors + learner_id];
2242         struct learner_runtime *l = &t->learners[learner_id];
2243         struct learner_statistics *stats = &p->learner_stats[learner_id];
2244         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2245         uint8_t *action_data;
2246         int done, hit;
2247
2248         /* Table. */
2249         time = rte_get_tsc_cycles();
2250
2251         done = rte_swx_table_learner_lookup(ts->obj,
2252                                             l->mailbox,
2253                                             time,
2254                                             l->key,
2255                                             &action_id,
2256                                             &action_data,
2257                                             &hit);
2258         if (!done) {
2259                 /* Thread. */
2260                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2261                       p->thread_id,
2262                       learner_id);
2263
2264                 thread_yield(p);
2265                 return;
2266         }
2267
2268         action_id = hit ? action_id : ts->default_action_id;
2269         action_data = hit ? action_data : ts->default_action_data;
2270         n_pkts_hit = stats->n_pkts_hit[hit];
2271         n_pkts_action = stats->n_pkts_action[action_id];
2272
2273         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2274               p->thread_id,
2275               learner_id,
2276               hit ? "hit" : "miss",
2277               (uint32_t)action_id);
2278
2279         t->action_id = action_id;
2280         t->structs[0] = action_data;
2281         t->hit = hit;
2282         t->learner_id = learner_id;
2283         t->time = time;
2284         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2285         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2286
2287         /* Thread. */
2288         thread_ip_action_call(p, t, action_id);
2289 }
2290
2291 static inline void
2292 instr_learner_af_exec(struct rte_swx_pipeline *p)
2293 {
2294         struct thread *t = &p->threads[p->thread_id];
2295         struct instruction *ip = t->ip;
2296         uint32_t learner_id = ip->table.table_id;
2297         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2298                 p->n_selectors + learner_id];
2299         struct learner_runtime *l = &t->learners[learner_id];
2300         struct learner_statistics *stats = &p->learner_stats[learner_id];
2301         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2302         uint8_t *action_data;
2303         action_func_t action_func;
2304         int done, hit;
2305
2306         /* Table. */
2307         time = rte_get_tsc_cycles();
2308
2309         done = rte_swx_table_learner_lookup(ts->obj,
2310                                             l->mailbox,
2311                                             time,
2312                                             l->key,
2313                                             &action_id,
2314                                             &action_data,
2315                                             &hit);
2316         if (!done) {
2317                 /* Thread. */
2318                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2319                       p->thread_id,
2320                       learner_id);
2321
2322                 thread_yield(p);
2323                 return;
2324         }
2325
2326         action_id = hit ? action_id : ts->default_action_id;
2327         action_data = hit ? action_data : ts->default_action_data;
2328         action_func = p->action_funcs[action_id];
2329         n_pkts_hit = stats->n_pkts_hit[hit];
2330         n_pkts_action = stats->n_pkts_action[action_id];
2331
2332         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2333               p->thread_id,
2334               learner_id,
2335               hit ? "hit" : "miss",
2336               (uint32_t)action_id);
2337
2338         t->action_id = action_id;
2339         t->structs[0] = action_data;
2340         t->hit = hit;
2341         t->learner_id = learner_id;
2342         t->time = time;
2343         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2344         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2345
2346         /* Thread. */
2347         thread_ip_action_call(p, t, action_id);
2348
2349         /* Action */
2350         action_func(p);
2351 }
2352
2353 /*
2354  * learn.
2355  */
2356 static struct action *
2357 action_find(struct rte_swx_pipeline *p, const char *name);
2358
2359 static int
2360 action_has_nbo_args(struct action *a);
2361
2362 static int
2363 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2364
2365 static int
2366 instr_learn_translate(struct rte_swx_pipeline *p,
2367                       struct action *action,
2368                       char **tokens,
2369                       int n_tokens,
2370                       struct instruction *instr,
2371                       struct instruction_data *data __rte_unused)
2372 {
2373         struct action *a;
2374         const char *mf_name;
2375         uint32_t mf_offset = 0;
2376
2377         CHECK(action, EINVAL);
2378         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2379
2380         a = action_find(p, tokens[1]);
2381         CHECK(a, EINVAL);
2382         CHECK(!action_has_nbo_args(a), EINVAL);
2383
2384         mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2385         CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2386
2387         if (mf_name) {
2388                 struct field *mf;
2389
2390                 mf = metadata_field_parse(p, mf_name);
2391                 CHECK(mf, EINVAL);
2392
2393                 mf_offset = mf->offset / 8;
2394         }
2395
2396         instr->type = INSTR_LEARNER_LEARN;
2397         instr->learn.action_id = a->id;
2398         instr->learn.mf_offset = mf_offset;
2399
2400         return 0;
2401 }
2402
2403 static inline void
2404 instr_learn_exec(struct rte_swx_pipeline *p)
2405 {
2406         struct thread *t = &p->threads[p->thread_id];
2407         struct instruction *ip = t->ip;
2408
2409         __instr_learn_exec(p, t, ip);
2410
2411         /* Thread. */
2412         thread_ip_inc(p);
2413 }
2414
2415 /*
2416  * forget.
2417  */
2418 static int
2419 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2420                        struct action *action,
2421                        char **tokens __rte_unused,
2422                        int n_tokens,
2423                        struct instruction *instr,
2424                        struct instruction_data *data __rte_unused)
2425 {
2426         CHECK(action, EINVAL);
2427         CHECK(n_tokens == 1, EINVAL);
2428
2429         instr->type = INSTR_LEARNER_FORGET;
2430
2431         return 0;
2432 }
2433
2434 static inline void
2435 instr_forget_exec(struct rte_swx_pipeline *p)
2436 {
2437         struct thread *t = &p->threads[p->thread_id];
2438         struct instruction *ip = t->ip;
2439
2440         __instr_forget_exec(p, t, ip);
2441
2442         /* Thread. */
2443         thread_ip_inc(p);
2444 }
2445
2446 /*
2447  * extern.
2448  */
2449 static int
2450 instr_extern_translate(struct rte_swx_pipeline *p,
2451                        struct action *action __rte_unused,
2452                        char **tokens,
2453                        int n_tokens,
2454                        struct instruction *instr,
2455                        struct instruction_data *data __rte_unused)
2456 {
2457         char *token = tokens[1];
2458
2459         CHECK(n_tokens == 2, EINVAL);
2460
2461         if (token[0] == 'e') {
2462                 struct extern_obj *obj;
2463                 struct extern_type_member_func *func;
2464
2465                 func = extern_obj_member_func_parse(p, token, &obj);
2466                 CHECK(func, EINVAL);
2467
2468                 instr->type = INSTR_EXTERN_OBJ;
2469                 instr->ext_obj.ext_obj_id = obj->id;
2470                 instr->ext_obj.func_id = func->id;
2471
2472                 return 0;
2473         }
2474
2475         if (token[0] == 'f') {
2476                 struct extern_func *func;
2477
2478                 func = extern_func_parse(p, token);
2479                 CHECK(func, EINVAL);
2480
2481                 instr->type = INSTR_EXTERN_FUNC;
2482                 instr->ext_func.ext_func_id = func->id;
2483
2484                 return 0;
2485         }
2486
2487         CHECK(0, EINVAL);
2488 }
2489
2490 static inline void
2491 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2492 {
2493         struct thread *t = &p->threads[p->thread_id];
2494         struct instruction *ip = t->ip;
2495         uint32_t done;
2496
2497         /* Extern object member function execute. */
2498         done = __instr_extern_obj_exec(p, t, ip);
2499
2500         /* Thread. */
2501         thread_ip_inc_cond(t, done);
2502         thread_yield_cond(p, done ^ 1);
2503 }
2504
2505 static inline void
2506 instr_extern_func_exec(struct rte_swx_pipeline *p)
2507 {
2508         struct thread *t = &p->threads[p->thread_id];
2509         struct instruction *ip = t->ip;
2510         uint32_t done;
2511
2512         /* Extern function execute. */
2513         done = __instr_extern_func_exec(p, t, ip);
2514
2515         /* Thread. */
2516         thread_ip_inc_cond(t, done);
2517         thread_yield_cond(p, done ^ 1);
2518 }
2519
2520 /*
2521  * mov.
2522  */
2523 static int
2524 instr_mov_translate(struct rte_swx_pipeline *p,
2525                     struct action *action,
2526                     char **tokens,
2527                     int n_tokens,
2528                     struct instruction *instr,
2529                     struct instruction_data *data __rte_unused)
2530 {
2531         char *dst = tokens[1], *src = tokens[2];
2532         struct field *fdst, *fsrc;
2533         uint64_t src_val;
2534         uint32_t dst_struct_id = 0, src_struct_id = 0;
2535
2536         CHECK(n_tokens == 3, EINVAL);
2537
2538         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2539         CHECK(fdst, EINVAL);
2540         CHECK(!fdst->var_size, EINVAL);
2541
2542         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2543         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2544         if (fsrc) {
2545                 CHECK(!fsrc->var_size, EINVAL);
2546
2547                 instr->type = INSTR_MOV;
2548                 if (dst[0] != 'h' && src[0] == 'h')
2549                         instr->type = INSTR_MOV_MH;
2550                 if (dst[0] == 'h' && src[0] != 'h')
2551                         instr->type = INSTR_MOV_HM;
2552                 if (dst[0] == 'h' && src[0] == 'h')
2553                         instr->type = INSTR_MOV_HH;
2554
2555                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2556                 instr->mov.dst.n_bits = fdst->n_bits;
2557                 instr->mov.dst.offset = fdst->offset / 8;
2558                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2559                 instr->mov.src.n_bits = fsrc->n_bits;
2560                 instr->mov.src.offset = fsrc->offset / 8;
2561                 return 0;
2562         }
2563
2564         /* MOV_I. */
2565         src_val = strtoull(src, &src, 0);
2566         CHECK(!src[0], EINVAL);
2567
2568         if (dst[0] == 'h')
2569                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2570
2571         instr->type = INSTR_MOV_I;
2572         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2573         instr->mov.dst.n_bits = fdst->n_bits;
2574         instr->mov.dst.offset = fdst->offset / 8;
2575         instr->mov.src_val = src_val;
2576         return 0;
2577 }
2578
2579 static inline void
2580 instr_mov_exec(struct rte_swx_pipeline *p)
2581 {
2582         struct thread *t = &p->threads[p->thread_id];
2583         struct instruction *ip = t->ip;
2584
2585         __instr_mov_exec(p, t, ip);
2586
2587         /* Thread. */
2588         thread_ip_inc(p);
2589 }
2590
2591 static inline void
2592 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2593 {
2594         struct thread *t = &p->threads[p->thread_id];
2595         struct instruction *ip = t->ip;
2596
2597         __instr_mov_mh_exec(p, t, ip);
2598
2599         /* Thread. */
2600         thread_ip_inc(p);
2601 }
2602
2603 static inline void
2604 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2605 {
2606         struct thread *t = &p->threads[p->thread_id];
2607         struct instruction *ip = t->ip;
2608
2609         __instr_mov_hm_exec(p, t, ip);
2610
2611         /* Thread. */
2612         thread_ip_inc(p);
2613 }
2614
2615 static inline void
2616 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2617 {
2618         struct thread *t = &p->threads[p->thread_id];
2619         struct instruction *ip = t->ip;
2620
2621         __instr_mov_hh_exec(p, t, ip);
2622
2623         /* Thread. */
2624         thread_ip_inc(p);
2625 }
2626
2627 static inline void
2628 instr_mov_i_exec(struct rte_swx_pipeline *p)
2629 {
2630         struct thread *t = &p->threads[p->thread_id];
2631         struct instruction *ip = t->ip;
2632
2633         __instr_mov_i_exec(p, t, ip);
2634
2635         /* Thread. */
2636         thread_ip_inc(p);
2637 }
2638
2639 /*
2640  * dma.
2641  */
2642 static inline void
2643 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2644 {
2645         struct thread *t = &p->threads[p->thread_id];
2646         struct instruction *ip = t->ip;
2647
2648         __instr_dma_ht_exec(p, t, ip);
2649
2650         /* Thread. */
2651         thread_ip_inc(p);
2652 }
2653
2654 static inline void
2655 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2656 {
2657         struct thread *t = &p->threads[p->thread_id];
2658         struct instruction *ip = t->ip;
2659
2660         __instr_dma_ht2_exec(p, t, ip);
2661
2662         /* Thread. */
2663         thread_ip_inc(p);
2664 }
2665
2666 static inline void
2667 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2668 {
2669         struct thread *t = &p->threads[p->thread_id];
2670         struct instruction *ip = t->ip;
2671
2672         __instr_dma_ht3_exec(p, t, ip);
2673
2674         /* Thread. */
2675         thread_ip_inc(p);
2676 }
2677
2678 static inline void
2679 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2680 {
2681         struct thread *t = &p->threads[p->thread_id];
2682         struct instruction *ip = t->ip;
2683
2684         __instr_dma_ht4_exec(p, t, ip);
2685
2686         /* Thread. */
2687         thread_ip_inc(p);
2688 }
2689
2690 static inline void
2691 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2692 {
2693         struct thread *t = &p->threads[p->thread_id];
2694         struct instruction *ip = t->ip;
2695
2696         __instr_dma_ht5_exec(p, t, ip);
2697
2698         /* Thread. */
2699         thread_ip_inc(p);
2700 }
2701
2702 static inline void
2703 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2704 {
2705         struct thread *t = &p->threads[p->thread_id];
2706         struct instruction *ip = t->ip;
2707
2708         __instr_dma_ht6_exec(p, t, ip);
2709
2710         /* Thread. */
2711         thread_ip_inc(p);
2712 }
2713
2714 static inline void
2715 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2716 {
2717         struct thread *t = &p->threads[p->thread_id];
2718         struct instruction *ip = t->ip;
2719
2720         __instr_dma_ht7_exec(p, t, ip);
2721
2722         /* Thread. */
2723         thread_ip_inc(p);
2724 }
2725
2726 static inline void
2727 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2728 {
2729         struct thread *t = &p->threads[p->thread_id];
2730         struct instruction *ip = t->ip;
2731
2732         __instr_dma_ht8_exec(p, t, ip);
2733
2734         /* Thread. */
2735         thread_ip_inc(p);
2736 }
2737
2738 /*
2739  * alu.
2740  */
2741 static int
2742 instr_alu_add_translate(struct rte_swx_pipeline *p,
2743                         struct action *action,
2744                         char **tokens,
2745                         int n_tokens,
2746                         struct instruction *instr,
2747                         struct instruction_data *data __rte_unused)
2748 {
2749         char *dst = tokens[1], *src = tokens[2];
2750         struct field *fdst, *fsrc;
2751         uint64_t src_val;
2752         uint32_t dst_struct_id = 0, src_struct_id = 0;
2753
2754         CHECK(n_tokens == 3, EINVAL);
2755
2756         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2757         CHECK(fdst, EINVAL);
2758         CHECK(!fdst->var_size, EINVAL);
2759
2760         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2761         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2762         if (fsrc) {
2763                 CHECK(!fsrc->var_size, EINVAL);
2764
2765                 instr->type = INSTR_ALU_ADD;
2766                 if (dst[0] == 'h' && src[0] != 'h')
2767                         instr->type = INSTR_ALU_ADD_HM;
2768                 if (dst[0] != 'h' && src[0] == 'h')
2769                         instr->type = INSTR_ALU_ADD_MH;
2770                 if (dst[0] == 'h' && src[0] == 'h')
2771                         instr->type = INSTR_ALU_ADD_HH;
2772
2773                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2774                 instr->alu.dst.n_bits = fdst->n_bits;
2775                 instr->alu.dst.offset = fdst->offset / 8;
2776                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2777                 instr->alu.src.n_bits = fsrc->n_bits;
2778                 instr->alu.src.offset = fsrc->offset / 8;
2779                 return 0;
2780         }
2781
2782         /* ADD_MI, ADD_HI. */
2783         src_val = strtoull(src, &src, 0);
2784         CHECK(!src[0], EINVAL);
2785
2786         instr->type = INSTR_ALU_ADD_MI;
2787         if (dst[0] == 'h')
2788                 instr->type = INSTR_ALU_ADD_HI;
2789
2790         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2791         instr->alu.dst.n_bits = fdst->n_bits;
2792         instr->alu.dst.offset = fdst->offset / 8;
2793         instr->alu.src_val = src_val;
2794         return 0;
2795 }
2796
2797 static int
2798 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2799                         struct action *action,
2800                         char **tokens,
2801                         int n_tokens,
2802                         struct instruction *instr,
2803                         struct instruction_data *data __rte_unused)
2804 {
2805         char *dst = tokens[1], *src = tokens[2];
2806         struct field *fdst, *fsrc;
2807         uint64_t src_val;
2808         uint32_t dst_struct_id = 0, src_struct_id = 0;
2809
2810         CHECK(n_tokens == 3, EINVAL);
2811
2812         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2813         CHECK(fdst, EINVAL);
2814         CHECK(!fdst->var_size, EINVAL);
2815
2816         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2817         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2818         if (fsrc) {
2819                 CHECK(!fsrc->var_size, EINVAL);
2820
2821                 instr->type = INSTR_ALU_SUB;
2822                 if (dst[0] == 'h' && src[0] != 'h')
2823                         instr->type = INSTR_ALU_SUB_HM;
2824                 if (dst[0] != 'h' && src[0] == 'h')
2825                         instr->type = INSTR_ALU_SUB_MH;
2826                 if (dst[0] == 'h' && src[0] == 'h')
2827                         instr->type = INSTR_ALU_SUB_HH;
2828
2829                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2830                 instr->alu.dst.n_bits = fdst->n_bits;
2831                 instr->alu.dst.offset = fdst->offset / 8;
2832                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2833                 instr->alu.src.n_bits = fsrc->n_bits;
2834                 instr->alu.src.offset = fsrc->offset / 8;
2835                 return 0;
2836         }
2837
2838         /* SUB_MI, SUB_HI. */
2839         src_val = strtoull(src, &src, 0);
2840         CHECK(!src[0], EINVAL);
2841
2842         instr->type = INSTR_ALU_SUB_MI;
2843         if (dst[0] == 'h')
2844                 instr->type = INSTR_ALU_SUB_HI;
2845
2846         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2847         instr->alu.dst.n_bits = fdst->n_bits;
2848         instr->alu.dst.offset = fdst->offset / 8;
2849         instr->alu.src_val = src_val;
2850         return 0;
2851 }
2852
2853 static int
2854 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2855                           struct action *action __rte_unused,
2856                           char **tokens,
2857                           int n_tokens,
2858                           struct instruction *instr,
2859                           struct instruction_data *data __rte_unused)
2860 {
2861         char *dst = tokens[1], *src = tokens[2];
2862         struct header *hdst, *hsrc;
2863         struct field *fdst, *fsrc;
2864
2865         CHECK(n_tokens == 3, EINVAL);
2866
2867         fdst = header_field_parse(p, dst, &hdst);
2868         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2869         CHECK(!fdst->var_size, EINVAL);
2870
2871         /* CKADD_FIELD. */
2872         fsrc = header_field_parse(p, src, &hsrc);
2873         if (fsrc) {
2874                 CHECK(!fsrc->var_size, EINVAL);
2875
2876                 instr->type = INSTR_ALU_CKADD_FIELD;
2877                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2878                 instr->alu.dst.n_bits = fdst->n_bits;
2879                 instr->alu.dst.offset = fdst->offset / 8;
2880                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2881                 instr->alu.src.n_bits = fsrc->n_bits;
2882                 instr->alu.src.offset = fsrc->offset / 8;
2883                 return 0;
2884         }
2885
2886         /* CKADD_STRUCT, CKADD_STRUCT20. */
2887         hsrc = header_parse(p, src);
2888         CHECK(hsrc, EINVAL);
2889         CHECK(!hsrc->st->var_size, EINVAL);
2890
2891         instr->type = INSTR_ALU_CKADD_STRUCT;
2892         if ((hsrc->st->n_bits / 8) == 20)
2893                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2894
2895         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2896         instr->alu.dst.n_bits = fdst->n_bits;
2897         instr->alu.dst.offset = fdst->offset / 8;
2898         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2899         instr->alu.src.n_bits = hsrc->st->n_bits;
2900         instr->alu.src.offset = 0; /* Unused. */
2901         return 0;
2902 }
2903
2904 static int
2905 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2906                           struct action *action __rte_unused,
2907                           char **tokens,
2908                           int n_tokens,
2909                           struct instruction *instr,
2910                           struct instruction_data *data __rte_unused)
2911 {
2912         char *dst = tokens[1], *src = tokens[2];
2913         struct header *hdst, *hsrc;
2914         struct field *fdst, *fsrc;
2915
2916         CHECK(n_tokens == 3, EINVAL);
2917
2918         fdst = header_field_parse(p, dst, &hdst);
2919         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2920         CHECK(!fdst->var_size, EINVAL);
2921
2922         fsrc = header_field_parse(p, src, &hsrc);
2923         CHECK(fsrc, EINVAL);
2924         CHECK(!fsrc->var_size, EINVAL);
2925
2926         instr->type = INSTR_ALU_CKSUB_FIELD;
2927         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2928         instr->alu.dst.n_bits = fdst->n_bits;
2929         instr->alu.dst.offset = fdst->offset / 8;
2930         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2931         instr->alu.src.n_bits = fsrc->n_bits;
2932         instr->alu.src.offset = fsrc->offset / 8;
2933         return 0;
2934 }
2935
2936 static int
2937 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2938                         struct action *action,
2939                         char **tokens,
2940                         int n_tokens,
2941                         struct instruction *instr,
2942                         struct instruction_data *data __rte_unused)
2943 {
2944         char *dst = tokens[1], *src = tokens[2];
2945         struct field *fdst, *fsrc;
2946         uint64_t src_val;
2947         uint32_t dst_struct_id = 0, src_struct_id = 0;
2948
2949         CHECK(n_tokens == 3, EINVAL);
2950
2951         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2952         CHECK(fdst, EINVAL);
2953         CHECK(!fdst->var_size, EINVAL);
2954
2955         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2956         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2957         if (fsrc) {
2958                 CHECK(!fsrc->var_size, EINVAL);
2959
2960                 instr->type = INSTR_ALU_SHL;
2961                 if (dst[0] == 'h' && src[0] != 'h')
2962                         instr->type = INSTR_ALU_SHL_HM;
2963                 if (dst[0] != 'h' && src[0] == 'h')
2964                         instr->type = INSTR_ALU_SHL_MH;
2965                 if (dst[0] == 'h' && src[0] == 'h')
2966                         instr->type = INSTR_ALU_SHL_HH;
2967
2968                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2969                 instr->alu.dst.n_bits = fdst->n_bits;
2970                 instr->alu.dst.offset = fdst->offset / 8;
2971                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2972                 instr->alu.src.n_bits = fsrc->n_bits;
2973                 instr->alu.src.offset = fsrc->offset / 8;
2974                 return 0;
2975         }
2976
2977         /* SHL_MI, SHL_HI. */
2978         src_val = strtoull(src, &src, 0);
2979         CHECK(!src[0], EINVAL);
2980
2981         instr->type = INSTR_ALU_SHL_MI;
2982         if (dst[0] == 'h')
2983                 instr->type = INSTR_ALU_SHL_HI;
2984
2985         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2986         instr->alu.dst.n_bits = fdst->n_bits;
2987         instr->alu.dst.offset = fdst->offset / 8;
2988         instr->alu.src_val = src_val;
2989         return 0;
2990 }
2991
2992 static int
2993 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2994                         struct action *action,
2995                         char **tokens,
2996                         int n_tokens,
2997                         struct instruction *instr,
2998                         struct instruction_data *data __rte_unused)
2999 {
3000         char *dst = tokens[1], *src = tokens[2];
3001         struct field *fdst, *fsrc;
3002         uint64_t src_val;
3003         uint32_t dst_struct_id = 0, src_struct_id = 0;
3004
3005         CHECK(n_tokens == 3, EINVAL);
3006
3007         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3008         CHECK(fdst, EINVAL);
3009         CHECK(!fdst->var_size, EINVAL);
3010
3011         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3012         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3013         if (fsrc) {
3014                 CHECK(!fsrc->var_size, EINVAL);
3015
3016                 instr->type = INSTR_ALU_SHR;
3017                 if (dst[0] == 'h' && src[0] != 'h')
3018                         instr->type = INSTR_ALU_SHR_HM;
3019                 if (dst[0] != 'h' && src[0] == 'h')
3020                         instr->type = INSTR_ALU_SHR_MH;
3021                 if (dst[0] == 'h' && src[0] == 'h')
3022                         instr->type = INSTR_ALU_SHR_HH;
3023
3024                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3025                 instr->alu.dst.n_bits = fdst->n_bits;
3026                 instr->alu.dst.offset = fdst->offset / 8;
3027                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3028                 instr->alu.src.n_bits = fsrc->n_bits;
3029                 instr->alu.src.offset = fsrc->offset / 8;
3030                 return 0;
3031         }
3032
3033         /* SHR_MI, SHR_HI. */
3034         src_val = strtoull(src, &src, 0);
3035         CHECK(!src[0], EINVAL);
3036
3037         instr->type = INSTR_ALU_SHR_MI;
3038         if (dst[0] == 'h')
3039                 instr->type = INSTR_ALU_SHR_HI;
3040
3041         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3042         instr->alu.dst.n_bits = fdst->n_bits;
3043         instr->alu.dst.offset = fdst->offset / 8;
3044         instr->alu.src_val = src_val;
3045         return 0;
3046 }
3047
3048 static int
3049 instr_alu_and_translate(struct rte_swx_pipeline *p,
3050                         struct action *action,
3051                         char **tokens,
3052                         int n_tokens,
3053                         struct instruction *instr,
3054                         struct instruction_data *data __rte_unused)
3055 {
3056         char *dst = tokens[1], *src = tokens[2];
3057         struct field *fdst, *fsrc;
3058         uint64_t src_val;
3059         uint32_t dst_struct_id = 0, src_struct_id = 0;
3060
3061         CHECK(n_tokens == 3, EINVAL);
3062
3063         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3064         CHECK(fdst, EINVAL);
3065         CHECK(!fdst->var_size, EINVAL);
3066
3067         /* AND, AND_MH, AND_HM, AND_HH. */
3068         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3069         if (fsrc) {
3070                 CHECK(!fsrc->var_size, EINVAL);
3071
3072                 instr->type = INSTR_ALU_AND;
3073                 if (dst[0] != 'h' && src[0] == 'h')
3074                         instr->type = INSTR_ALU_AND_MH;
3075                 if (dst[0] == 'h' && src[0] != 'h')
3076                         instr->type = INSTR_ALU_AND_HM;
3077                 if (dst[0] == 'h' && src[0] == 'h')
3078                         instr->type = INSTR_ALU_AND_HH;
3079
3080                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3081                 instr->alu.dst.n_bits = fdst->n_bits;
3082                 instr->alu.dst.offset = fdst->offset / 8;
3083                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3084                 instr->alu.src.n_bits = fsrc->n_bits;
3085                 instr->alu.src.offset = fsrc->offset / 8;
3086                 return 0;
3087         }
3088
3089         /* AND_I. */
3090         src_val = strtoull(src, &src, 0);
3091         CHECK(!src[0], EINVAL);
3092
3093         if (dst[0] == 'h')
3094                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3095
3096         instr->type = INSTR_ALU_AND_I;
3097         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3098         instr->alu.dst.n_bits = fdst->n_bits;
3099         instr->alu.dst.offset = fdst->offset / 8;
3100         instr->alu.src_val = src_val;
3101         return 0;
3102 }
3103
3104 static int
3105 instr_alu_or_translate(struct rte_swx_pipeline *p,
3106                        struct action *action,
3107                        char **tokens,
3108                        int n_tokens,
3109                        struct instruction *instr,
3110                        struct instruction_data *data __rte_unused)
3111 {
3112         char *dst = tokens[1], *src = tokens[2];
3113         struct field *fdst, *fsrc;
3114         uint64_t src_val;
3115         uint32_t dst_struct_id = 0, src_struct_id = 0;
3116
3117         CHECK(n_tokens == 3, EINVAL);
3118
3119         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3120         CHECK(fdst, EINVAL);
3121         CHECK(!fdst->var_size, EINVAL);
3122
3123         /* OR, OR_MH, OR_HM, OR_HH. */
3124         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3125         if (fsrc) {
3126                 CHECK(!fsrc->var_size, EINVAL);
3127
3128                 instr->type = INSTR_ALU_OR;
3129                 if (dst[0] != 'h' && src[0] == 'h')
3130                         instr->type = INSTR_ALU_OR_MH;
3131                 if (dst[0] == 'h' && src[0] != 'h')
3132                         instr->type = INSTR_ALU_OR_HM;
3133                 if (dst[0] == 'h' && src[0] == 'h')
3134                         instr->type = INSTR_ALU_OR_HH;
3135
3136                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3137                 instr->alu.dst.n_bits = fdst->n_bits;
3138                 instr->alu.dst.offset = fdst->offset / 8;
3139                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3140                 instr->alu.src.n_bits = fsrc->n_bits;
3141                 instr->alu.src.offset = fsrc->offset / 8;
3142                 return 0;
3143         }
3144
3145         /* OR_I. */
3146         src_val = strtoull(src, &src, 0);
3147         CHECK(!src[0], EINVAL);
3148
3149         if (dst[0] == 'h')
3150                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3151
3152         instr->type = INSTR_ALU_OR_I;
3153         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3154         instr->alu.dst.n_bits = fdst->n_bits;
3155         instr->alu.dst.offset = fdst->offset / 8;
3156         instr->alu.src_val = src_val;
3157         return 0;
3158 }
3159
3160 static int
3161 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3162                         struct action *action,
3163                         char **tokens,
3164                         int n_tokens,
3165                         struct instruction *instr,
3166                         struct instruction_data *data __rte_unused)
3167 {
3168         char *dst = tokens[1], *src = tokens[2];
3169         struct field *fdst, *fsrc;
3170         uint64_t src_val;
3171         uint32_t dst_struct_id = 0, src_struct_id = 0;
3172
3173         CHECK(n_tokens == 3, EINVAL);
3174
3175         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3176         CHECK(fdst, EINVAL);
3177         CHECK(!fdst->var_size, EINVAL);
3178
3179         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3180         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3181         if (fsrc) {
3182                 CHECK(!fsrc->var_size, EINVAL);
3183
3184                 instr->type = INSTR_ALU_XOR;
3185                 if (dst[0] != 'h' && src[0] == 'h')
3186                         instr->type = INSTR_ALU_XOR_MH;
3187                 if (dst[0] == 'h' && src[0] != 'h')
3188                         instr->type = INSTR_ALU_XOR_HM;
3189                 if (dst[0] == 'h' && src[0] == 'h')
3190                         instr->type = INSTR_ALU_XOR_HH;
3191
3192                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3193                 instr->alu.dst.n_bits = fdst->n_bits;
3194                 instr->alu.dst.offset = fdst->offset / 8;
3195                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3196                 instr->alu.src.n_bits = fsrc->n_bits;
3197                 instr->alu.src.offset = fsrc->offset / 8;
3198                 return 0;
3199         }
3200
3201         /* XOR_I. */
3202         src_val = strtoull(src, &src, 0);
3203         CHECK(!src[0], EINVAL);
3204
3205         if (dst[0] == 'h')
3206                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3207
3208         instr->type = INSTR_ALU_XOR_I;
3209         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3210         instr->alu.dst.n_bits = fdst->n_bits;
3211         instr->alu.dst.offset = fdst->offset / 8;
3212         instr->alu.src_val = src_val;
3213         return 0;
3214 }
3215
3216 static inline void
3217 instr_alu_add_exec(struct rte_swx_pipeline *p)
3218 {
3219         struct thread *t = &p->threads[p->thread_id];
3220         struct instruction *ip = t->ip;
3221
3222         /* Structs */
3223         __instr_alu_add_exec(p, t, ip);
3224
3225         /* Thread. */
3226         thread_ip_inc(p);
3227 }
3228
3229 static inline void
3230 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3231 {
3232         struct thread *t = &p->threads[p->thread_id];
3233         struct instruction *ip = t->ip;
3234
3235         /* Structs. */
3236         __instr_alu_add_mh_exec(p, t, ip);
3237
3238         /* Thread. */
3239         thread_ip_inc(p);
3240 }
3241
3242 static inline void
3243 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3244 {
3245         struct thread *t = &p->threads[p->thread_id];
3246         struct instruction *ip = t->ip;
3247
3248         /* Structs. */
3249         __instr_alu_add_hm_exec(p, t, ip);
3250
3251         /* Thread. */
3252         thread_ip_inc(p);
3253 }
3254
3255 static inline void
3256 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3257 {
3258         struct thread *t = &p->threads[p->thread_id];
3259         struct instruction *ip = t->ip;
3260
3261         /* Structs. */
3262         __instr_alu_add_hh_exec(p, t, ip);
3263
3264         /* Thread. */
3265         thread_ip_inc(p);
3266 }
3267
3268 static inline void
3269 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3270 {
3271         struct thread *t = &p->threads[p->thread_id];
3272         struct instruction *ip = t->ip;
3273
3274         /* Structs. */
3275         __instr_alu_add_mi_exec(p, t, ip);
3276
3277         /* Thread. */
3278         thread_ip_inc(p);
3279 }
3280
3281 static inline void
3282 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3283 {
3284         struct thread *t = &p->threads[p->thread_id];
3285         struct instruction *ip = t->ip;
3286
3287         /* Structs. */
3288         __instr_alu_add_hi_exec(p, t, ip);
3289
3290         /* Thread. */
3291         thread_ip_inc(p);
3292 }
3293
3294 static inline void
3295 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3296 {
3297         struct thread *t = &p->threads[p->thread_id];
3298         struct instruction *ip = t->ip;
3299
3300         /* Structs. */
3301         __instr_alu_sub_exec(p, t, ip);
3302
3303         /* Thread. */
3304         thread_ip_inc(p);
3305 }
3306
3307 static inline void
3308 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3309 {
3310         struct thread *t = &p->threads[p->thread_id];
3311         struct instruction *ip = t->ip;
3312
3313         /* Structs. */
3314         __instr_alu_sub_mh_exec(p, t, ip);
3315
3316         /* Thread. */
3317         thread_ip_inc(p);
3318 }
3319
3320 static inline void
3321 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3322 {
3323         struct thread *t = &p->threads[p->thread_id];
3324         struct instruction *ip = t->ip;
3325
3326         /* Structs. */
3327         __instr_alu_sub_hm_exec(p, t, ip);
3328
3329         /* Thread. */
3330         thread_ip_inc(p);
3331 }
3332
3333 static inline void
3334 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3335 {
3336         struct thread *t = &p->threads[p->thread_id];
3337         struct instruction *ip = t->ip;
3338
3339         /* Structs. */
3340         __instr_alu_sub_hh_exec(p, t, ip);
3341
3342         /* Thread. */
3343         thread_ip_inc(p);
3344 }
3345
3346 static inline void
3347 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3348 {
3349         struct thread *t = &p->threads[p->thread_id];
3350         struct instruction *ip = t->ip;
3351
3352         /* Structs. */
3353         __instr_alu_sub_mi_exec(p, t, ip);
3354
3355         /* Thread. */
3356         thread_ip_inc(p);
3357 }
3358
3359 static inline void
3360 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3361 {
3362         struct thread *t = &p->threads[p->thread_id];
3363         struct instruction *ip = t->ip;
3364
3365         /* Structs. */
3366         __instr_alu_sub_hi_exec(p, t, ip);
3367
3368         /* Thread. */
3369         thread_ip_inc(p);
3370 }
3371
3372 static inline void
3373 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3374 {
3375         struct thread *t = &p->threads[p->thread_id];
3376         struct instruction *ip = t->ip;
3377
3378         /* Structs. */
3379         __instr_alu_shl_exec(p, t, ip);
3380
3381         /* Thread. */
3382         thread_ip_inc(p);
3383 }
3384
3385 static inline void
3386 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3387 {
3388         struct thread *t = &p->threads[p->thread_id];
3389         struct instruction *ip = t->ip;
3390
3391         /* Structs. */
3392         __instr_alu_shl_mh_exec(p, t, ip);
3393
3394         /* Thread. */
3395         thread_ip_inc(p);
3396 }
3397
3398 static inline void
3399 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3400 {
3401         struct thread *t = &p->threads[p->thread_id];
3402         struct instruction *ip = t->ip;
3403
3404         /* Structs. */
3405         __instr_alu_shl_hm_exec(p, t, ip);
3406
3407         /* Thread. */
3408         thread_ip_inc(p);
3409 }
3410
3411 static inline void
3412 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3413 {
3414         struct thread *t = &p->threads[p->thread_id];
3415         struct instruction *ip = t->ip;
3416
3417         /* Structs. */
3418         __instr_alu_shl_hh_exec(p, t, ip);
3419
3420         /* Thread. */
3421         thread_ip_inc(p);
3422 }
3423
3424 static inline void
3425 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3426 {
3427         struct thread *t = &p->threads[p->thread_id];
3428         struct instruction *ip = t->ip;
3429
3430         /* Structs. */
3431         __instr_alu_shl_mi_exec(p, t, ip);
3432
3433         /* Thread. */
3434         thread_ip_inc(p);
3435 }
3436
3437 static inline void
3438 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3439 {
3440         struct thread *t = &p->threads[p->thread_id];
3441         struct instruction *ip = t->ip;
3442
3443         /* Structs. */
3444         __instr_alu_shl_hi_exec(p, t, ip);
3445
3446         /* Thread. */
3447         thread_ip_inc(p);
3448 }
3449
3450 static inline void
3451 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3452 {
3453         struct thread *t = &p->threads[p->thread_id];
3454         struct instruction *ip = t->ip;
3455
3456         /* Structs. */
3457         __instr_alu_shr_exec(p, t, ip);
3458
3459         /* Thread. */
3460         thread_ip_inc(p);
3461 }
3462
3463 static inline void
3464 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3465 {
3466         struct thread *t = &p->threads[p->thread_id];
3467         struct instruction *ip = t->ip;
3468
3469         /* Structs. */
3470         __instr_alu_shr_mh_exec(p, t, ip);
3471
3472         /* Thread. */
3473         thread_ip_inc(p);
3474 }
3475
3476 static inline void
3477 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3478 {
3479         struct thread *t = &p->threads[p->thread_id];
3480         struct instruction *ip = t->ip;
3481
3482         /* Structs. */
3483         __instr_alu_shr_hm_exec(p, t, ip);
3484
3485         /* Thread. */
3486         thread_ip_inc(p);
3487 }
3488
3489 static inline void
3490 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3491 {
3492         struct thread *t = &p->threads[p->thread_id];
3493         struct instruction *ip = t->ip;
3494
3495         /* Structs. */
3496         __instr_alu_shr_hh_exec(p, t, ip);
3497
3498         /* Thread. */
3499         thread_ip_inc(p);
3500 }
3501
3502 static inline void
3503 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3504 {
3505         struct thread *t = &p->threads[p->thread_id];
3506         struct instruction *ip = t->ip;
3507
3508         /* Structs. */
3509         __instr_alu_shr_mi_exec(p, t, ip);
3510
3511         /* Thread. */
3512         thread_ip_inc(p);
3513 }
3514
3515 static inline void
3516 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3517 {
3518         struct thread *t = &p->threads[p->thread_id];
3519         struct instruction *ip = t->ip;
3520
3521         /* Structs. */
3522         __instr_alu_shr_hi_exec(p, t, ip);
3523
3524         /* Thread. */
3525         thread_ip_inc(p);
3526 }
3527
3528 static inline void
3529 instr_alu_and_exec(struct rte_swx_pipeline *p)
3530 {
3531         struct thread *t = &p->threads[p->thread_id];
3532         struct instruction *ip = t->ip;
3533
3534         /* Structs. */
3535         __instr_alu_and_exec(p, t, ip);
3536
3537         /* Thread. */
3538         thread_ip_inc(p);
3539 }
3540
3541 static inline void
3542 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3543 {
3544         struct thread *t = &p->threads[p->thread_id];
3545         struct instruction *ip = t->ip;
3546
3547         /* Structs. */
3548         __instr_alu_and_mh_exec(p, t, ip);
3549
3550         /* Thread. */
3551         thread_ip_inc(p);
3552 }
3553
3554 static inline void
3555 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3556 {
3557         struct thread *t = &p->threads[p->thread_id];
3558         struct instruction *ip = t->ip;
3559
3560         /* Structs. */
3561         __instr_alu_and_hm_exec(p, t, ip);
3562
3563         /* Thread. */
3564         thread_ip_inc(p);
3565 }
3566
3567 static inline void
3568 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3569 {
3570         struct thread *t = &p->threads[p->thread_id];
3571         struct instruction *ip = t->ip;
3572
3573         /* Structs. */
3574         __instr_alu_and_hh_exec(p, t, ip);
3575
3576         /* Thread. */
3577         thread_ip_inc(p);
3578 }
3579
3580 static inline void
3581 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3582 {
3583         struct thread *t = &p->threads[p->thread_id];
3584         struct instruction *ip = t->ip;
3585
3586         /* Structs. */
3587         __instr_alu_and_i_exec(p, t, ip);
3588
3589         /* Thread. */
3590         thread_ip_inc(p);
3591 }
3592
3593 static inline void
3594 instr_alu_or_exec(struct rte_swx_pipeline *p)
3595 {
3596         struct thread *t = &p->threads[p->thread_id];
3597         struct instruction *ip = t->ip;
3598
3599         /* Structs. */
3600         __instr_alu_or_exec(p, t, ip);
3601
3602         /* Thread. */
3603         thread_ip_inc(p);
3604 }
3605
3606 static inline void
3607 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3608 {
3609         struct thread *t = &p->threads[p->thread_id];
3610         struct instruction *ip = t->ip;
3611
3612         /* Structs. */
3613         __instr_alu_or_mh_exec(p, t, ip);
3614
3615         /* Thread. */
3616         thread_ip_inc(p);
3617 }
3618
3619 static inline void
3620 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3621 {
3622         struct thread *t = &p->threads[p->thread_id];
3623         struct instruction *ip = t->ip;
3624
3625         /* Structs. */
3626         __instr_alu_or_hm_exec(p, t, ip);
3627
3628         /* Thread. */
3629         thread_ip_inc(p);
3630 }
3631
3632 static inline void
3633 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3634 {
3635         struct thread *t = &p->threads[p->thread_id];
3636         struct instruction *ip = t->ip;
3637
3638         /* Structs. */
3639         __instr_alu_or_hh_exec(p, t, ip);
3640
3641         /* Thread. */
3642         thread_ip_inc(p);
3643 }
3644
3645 static inline void
3646 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3647 {
3648         struct thread *t = &p->threads[p->thread_id];
3649         struct instruction *ip = t->ip;
3650
3651         /* Structs. */
3652         __instr_alu_or_i_exec(p, t, ip);
3653
3654         /* Thread. */
3655         thread_ip_inc(p);
3656 }
3657
3658 static inline void
3659 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3660 {
3661         struct thread *t = &p->threads[p->thread_id];
3662         struct instruction *ip = t->ip;
3663
3664         /* Structs. */
3665         __instr_alu_xor_exec(p, t, ip);
3666
3667         /* Thread. */
3668         thread_ip_inc(p);
3669 }
3670
3671 static inline void
3672 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3673 {
3674         struct thread *t = &p->threads[p->thread_id];
3675         struct instruction *ip = t->ip;
3676
3677         /* Structs. */
3678         __instr_alu_xor_mh_exec(p, t, ip);
3679
3680         /* Thread. */
3681         thread_ip_inc(p);
3682 }
3683
3684 static inline void
3685 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3686 {
3687         struct thread *t = &p->threads[p->thread_id];
3688         struct instruction *ip = t->ip;
3689
3690         /* Structs. */
3691         __instr_alu_xor_hm_exec(p, t, ip);
3692
3693         /* Thread. */
3694         thread_ip_inc(p);
3695 }
3696
3697 static inline void
3698 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3699 {
3700         struct thread *t = &p->threads[p->thread_id];
3701         struct instruction *ip = t->ip;
3702
3703         /* Structs. */
3704         __instr_alu_xor_hh_exec(p, t, ip);
3705
3706         /* Thread. */
3707         thread_ip_inc(p);
3708 }
3709
3710 static inline void
3711 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3712 {
3713         struct thread *t = &p->threads[p->thread_id];
3714         struct instruction *ip = t->ip;
3715
3716         /* Structs. */
3717         __instr_alu_xor_i_exec(p, t, ip);
3718
3719         /* Thread. */
3720         thread_ip_inc(p);
3721 }
3722
3723 static inline void
3724 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3725 {
3726         struct thread *t = &p->threads[p->thread_id];
3727         struct instruction *ip = t->ip;
3728
3729         /* Structs. */
3730         __instr_alu_ckadd_field_exec(p, t, ip);
3731
3732         /* Thread. */
3733         thread_ip_inc(p);
3734 }
3735
3736 static inline void
3737 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3738 {
3739         struct thread *t = &p->threads[p->thread_id];
3740         struct instruction *ip = t->ip;
3741
3742         /* Structs. */
3743         __instr_alu_cksub_field_exec(p, t, ip);
3744
3745         /* Thread. */
3746         thread_ip_inc(p);
3747 }
3748
3749 static inline void
3750 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3751 {
3752         struct thread *t = &p->threads[p->thread_id];
3753         struct instruction *ip = t->ip;
3754
3755         /* Structs. */
3756         __instr_alu_ckadd_struct20_exec(p, t, ip);
3757
3758         /* Thread. */
3759         thread_ip_inc(p);
3760 }
3761
3762 static inline void
3763 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3764 {
3765         struct thread *t = &p->threads[p->thread_id];
3766         struct instruction *ip = t->ip;
3767
3768         /* Structs. */
3769         __instr_alu_ckadd_struct_exec(p, t, ip);
3770
3771         /* Thread. */
3772         thread_ip_inc(p);
3773 }
3774
3775 /*
3776  * Register array.
3777  */
3778 static struct regarray *
3779 regarray_find(struct rte_swx_pipeline *p, const char *name);
3780
3781 static int
3782 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3783                       struct action *action,
3784                       char **tokens,
3785                       int n_tokens,
3786                       struct instruction *instr,
3787                       struct instruction_data *data __rte_unused)
3788 {
3789         char *regarray = tokens[1], *idx = tokens[2];
3790         struct regarray *r;
3791         struct field *fidx;
3792         uint32_t idx_struct_id, idx_val;
3793
3794         CHECK(n_tokens == 3, EINVAL);
3795
3796         r = regarray_find(p, regarray);
3797         CHECK(r, EINVAL);
3798
3799         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3800         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3801         if (fidx) {
3802                 CHECK(!fidx->var_size, EINVAL);
3803
3804                 instr->type = INSTR_REGPREFETCH_RM;
3805                 if (idx[0] == 'h')
3806                         instr->type = INSTR_REGPREFETCH_RH;
3807
3808                 instr->regarray.regarray_id = r->id;
3809                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3810                 instr->regarray.idx.n_bits = fidx->n_bits;
3811                 instr->regarray.idx.offset = fidx->offset / 8;
3812                 instr->regarray.dstsrc_val = 0; /* Unused. */
3813                 return 0;
3814         }
3815
3816         /* REGPREFETCH_RI. */
3817         idx_val = strtoul(idx, &idx, 0);
3818         CHECK(!idx[0], EINVAL);
3819
3820         instr->type = INSTR_REGPREFETCH_RI;
3821         instr->regarray.regarray_id = r->id;
3822         instr->regarray.idx_val = idx_val;
3823         instr->regarray.dstsrc_val = 0; /* Unused. */
3824         return 0;
3825 }
3826
3827 static int
3828 instr_regrd_translate(struct rte_swx_pipeline *p,
3829                       struct action *action,
3830                       char **tokens,
3831                       int n_tokens,
3832                       struct instruction *instr,
3833                       struct instruction_data *data __rte_unused)
3834 {
3835         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3836         struct regarray *r;
3837         struct field *fdst, *fidx;
3838         uint32_t dst_struct_id, idx_struct_id, idx_val;
3839
3840         CHECK(n_tokens == 4, EINVAL);
3841
3842         r = regarray_find(p, regarray);
3843         CHECK(r, EINVAL);
3844
3845         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3846         CHECK(fdst, EINVAL);
3847         CHECK(!fdst->var_size, EINVAL);
3848
3849         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3850         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3851         if (fidx) {
3852                 CHECK(!fidx->var_size, EINVAL);
3853
3854                 instr->type = INSTR_REGRD_MRM;
3855                 if (dst[0] == 'h' && idx[0] != 'h')
3856                         instr->type = INSTR_REGRD_HRM;
3857                 if (dst[0] != 'h' && idx[0] == 'h')
3858                         instr->type = INSTR_REGRD_MRH;
3859                 if (dst[0] == 'h' && idx[0] == 'h')
3860                         instr->type = INSTR_REGRD_HRH;
3861
3862                 instr->regarray.regarray_id = r->id;
3863                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3864                 instr->regarray.idx.n_bits = fidx->n_bits;
3865                 instr->regarray.idx.offset = fidx->offset / 8;
3866                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3867                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3868                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3869                 return 0;
3870         }
3871
3872         /* REGRD_MRI, REGRD_HRI. */
3873         idx_val = strtoul(idx, &idx, 0);
3874         CHECK(!idx[0], EINVAL);
3875
3876         instr->type = INSTR_REGRD_MRI;
3877         if (dst[0] == 'h')
3878                 instr->type = INSTR_REGRD_HRI;
3879
3880         instr->regarray.regarray_id = r->id;
3881         instr->regarray.idx_val = idx_val;
3882         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3883         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3884         instr->regarray.dstsrc.offset = fdst->offset / 8;
3885         return 0;
3886 }
3887
3888 static int
3889 instr_regwr_translate(struct rte_swx_pipeline *p,
3890                       struct action *action,
3891                       char **tokens,
3892                       int n_tokens,
3893                       struct instruction *instr,
3894                       struct instruction_data *data __rte_unused)
3895 {
3896         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3897         struct regarray *r;
3898         struct field *fidx, *fsrc;
3899         uint64_t src_val;
3900         uint32_t idx_struct_id, idx_val, src_struct_id;
3901
3902         CHECK(n_tokens == 4, EINVAL);
3903
3904         r = regarray_find(p, regarray);
3905         CHECK(r, EINVAL);
3906
3907         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3908         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3909         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3910         if (fidx && fsrc) {
3911                 CHECK(!fidx->var_size, EINVAL);
3912                 CHECK(!fsrc->var_size, EINVAL);
3913
3914                 instr->type = INSTR_REGWR_RMM;
3915                 if (idx[0] == 'h' && src[0] != 'h')
3916                         instr->type = INSTR_REGWR_RHM;
3917                 if (idx[0] != 'h' && src[0] == 'h')
3918                         instr->type = INSTR_REGWR_RMH;
3919                 if (idx[0] == 'h' && src[0] == 'h')
3920                         instr->type = INSTR_REGWR_RHH;
3921
3922                 instr->regarray.regarray_id = r->id;
3923                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3924                 instr->regarray.idx.n_bits = fidx->n_bits;
3925                 instr->regarray.idx.offset = fidx->offset / 8;
3926                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3927                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3928                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3929                 return 0;
3930         }
3931
3932         /* REGWR_RHI, REGWR_RMI. */
3933         if (fidx && !fsrc) {
3934                 CHECK(!fidx->var_size, EINVAL);
3935
3936                 src_val = strtoull(src, &src, 0);
3937                 CHECK(!src[0], EINVAL);
3938
3939                 instr->type = INSTR_REGWR_RMI;
3940                 if (idx[0] == 'h')
3941                         instr->type = INSTR_REGWR_RHI;
3942
3943                 instr->regarray.regarray_id = r->id;
3944                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3945                 instr->regarray.idx.n_bits = fidx->n_bits;
3946                 instr->regarray.idx.offset = fidx->offset / 8;
3947                 instr->regarray.dstsrc_val = src_val;
3948                 return 0;
3949         }
3950
3951         /* REGWR_RIH, REGWR_RIM. */
3952         if (!fidx && fsrc) {
3953                 idx_val = strtoul(idx, &idx, 0);
3954                 CHECK(!idx[0], EINVAL);
3955
3956                 CHECK(!fsrc->var_size, EINVAL);
3957
3958                 instr->type = INSTR_REGWR_RIM;
3959                 if (src[0] == 'h')
3960                         instr->type = INSTR_REGWR_RIH;
3961
3962                 instr->regarray.regarray_id = r->id;
3963                 instr->regarray.idx_val = idx_val;
3964                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3965                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3966                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3967                 return 0;
3968         }
3969
3970         /* REGWR_RII. */
3971         src_val = strtoull(src, &src, 0);
3972         CHECK(!src[0], EINVAL);
3973
3974         idx_val = strtoul(idx, &idx, 0);
3975         CHECK(!idx[0], EINVAL);
3976
3977         instr->type = INSTR_REGWR_RII;
3978         instr->regarray.idx_val = idx_val;
3979         instr->regarray.dstsrc_val = src_val;
3980
3981         return 0;
3982 }
3983
3984 static int
3985 instr_regadd_translate(struct rte_swx_pipeline *p,
3986                        struct action *action,
3987                        char **tokens,
3988                        int n_tokens,
3989                        struct instruction *instr,
3990                        struct instruction_data *data __rte_unused)
3991 {
3992         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3993         struct regarray *r;
3994         struct field *fidx, *fsrc;
3995         uint64_t src_val;
3996         uint32_t idx_struct_id, idx_val, src_struct_id;
3997
3998         CHECK(n_tokens == 4, EINVAL);
3999
4000         r = regarray_find(p, regarray);
4001         CHECK(r, EINVAL);
4002
4003         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4004         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4005         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4006         if (fidx && fsrc) {
4007                 CHECK(!fidx->var_size, EINVAL);
4008                 CHECK(!fsrc->var_size, EINVAL);
4009
4010                 instr->type = INSTR_REGADD_RMM;
4011                 if (idx[0] == 'h' && src[0] != 'h')
4012                         instr->type = INSTR_REGADD_RHM;
4013                 if (idx[0] != 'h' && src[0] == 'h')
4014                         instr->type = INSTR_REGADD_RMH;
4015                 if (idx[0] == 'h' && src[0] == 'h')
4016                         instr->type = INSTR_REGADD_RHH;
4017
4018                 instr->regarray.regarray_id = r->id;
4019                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4020                 instr->regarray.idx.n_bits = fidx->n_bits;
4021                 instr->regarray.idx.offset = fidx->offset / 8;
4022                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4023                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4024                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4025                 return 0;
4026         }
4027
4028         /* REGADD_RHI, REGADD_RMI. */
4029         if (fidx && !fsrc) {
4030                 CHECK(!fidx->var_size, EINVAL);
4031
4032                 src_val = strtoull(src, &src, 0);
4033                 CHECK(!src[0], EINVAL);
4034
4035                 instr->type = INSTR_REGADD_RMI;
4036                 if (idx[0] == 'h')
4037                         instr->type = INSTR_REGADD_RHI;
4038
4039                 instr->regarray.regarray_id = r->id;
4040                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4041                 instr->regarray.idx.n_bits = fidx->n_bits;
4042                 instr->regarray.idx.offset = fidx->offset / 8;
4043                 instr->regarray.dstsrc_val = src_val;
4044                 return 0;
4045         }
4046
4047         /* REGADD_RIH, REGADD_RIM. */
4048         if (!fidx && fsrc) {
4049                 idx_val = strtoul(idx, &idx, 0);
4050                 CHECK(!idx[0], EINVAL);
4051
4052                 CHECK(!fsrc->var_size, EINVAL);
4053
4054                 instr->type = INSTR_REGADD_RIM;
4055                 if (src[0] == 'h')
4056                         instr->type = INSTR_REGADD_RIH;
4057
4058                 instr->regarray.regarray_id = r->id;
4059                 instr->regarray.idx_val = idx_val;
4060                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4061                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4062                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4063                 return 0;
4064         }
4065
4066         /* REGADD_RII. */
4067         src_val = strtoull(src, &src, 0);
4068         CHECK(!src[0], EINVAL);
4069
4070         idx_val = strtoul(idx, &idx, 0);
4071         CHECK(!idx[0], EINVAL);
4072
4073         instr->type = INSTR_REGADD_RII;
4074         instr->regarray.idx_val = idx_val;
4075         instr->regarray.dstsrc_val = src_val;
4076         return 0;
4077 }
4078
4079 static inline void
4080 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4081 {
4082         struct thread *t = &p->threads[p->thread_id];
4083         struct instruction *ip = t->ip;
4084
4085         /* Structs. */
4086         __instr_regprefetch_rh_exec(p, t, ip);
4087
4088         /* Thread. */
4089         thread_ip_inc(p);
4090 }
4091
4092 static inline void
4093 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4094 {
4095         struct thread *t = &p->threads[p->thread_id];
4096         struct instruction *ip = t->ip;
4097
4098         /* Structs. */
4099         __instr_regprefetch_rm_exec(p, t, ip);
4100
4101         /* Thread. */
4102         thread_ip_inc(p);
4103 }
4104
4105 static inline void
4106 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4107 {
4108         struct thread *t = &p->threads[p->thread_id];
4109         struct instruction *ip = t->ip;
4110
4111         /* Structs. */
4112         __instr_regprefetch_ri_exec(p, t, ip);
4113
4114         /* Thread. */
4115         thread_ip_inc(p);
4116 }
4117
4118 static inline void
4119 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4120 {
4121         struct thread *t = &p->threads[p->thread_id];
4122         struct instruction *ip = t->ip;
4123
4124         /* Structs. */
4125         __instr_regrd_hrh_exec(p, t, ip);
4126
4127         /* Thread. */
4128         thread_ip_inc(p);
4129 }
4130
4131 static inline void
4132 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4133 {
4134         struct thread *t = &p->threads[p->thread_id];
4135         struct instruction *ip = t->ip;
4136
4137         /* Structs. */
4138         __instr_regrd_hrm_exec(p, t, ip);
4139
4140         /* Thread. */
4141         thread_ip_inc(p);
4142 }
4143
4144 static inline void
4145 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4146 {
4147         struct thread *t = &p->threads[p->thread_id];
4148         struct instruction *ip = t->ip;
4149
4150         /* Structs. */
4151         __instr_regrd_mrh_exec(p, t, ip);
4152
4153         /* Thread. */
4154         thread_ip_inc(p);
4155 }
4156
4157 static inline void
4158 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4159 {
4160         struct thread *t = &p->threads[p->thread_id];
4161         struct instruction *ip = t->ip;
4162
4163         /* Structs. */
4164         __instr_regrd_mrm_exec(p, t, ip);
4165
4166         /* Thread. */
4167         thread_ip_inc(p);
4168 }
4169
4170 static inline void
4171 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4172 {
4173         struct thread *t = &p->threads[p->thread_id];
4174         struct instruction *ip = t->ip;
4175
4176         /* Structs. */
4177         __instr_regrd_hri_exec(p, t, ip);
4178
4179         /* Thread. */
4180         thread_ip_inc(p);
4181 }
4182
4183 static inline void
4184 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4185 {
4186         struct thread *t = &p->threads[p->thread_id];
4187         struct instruction *ip = t->ip;
4188
4189         /* Structs. */
4190         __instr_regrd_mri_exec(p, t, ip);
4191
4192         /* Thread. */
4193         thread_ip_inc(p);
4194 }
4195
4196 static inline void
4197 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4198 {
4199         struct thread *t = &p->threads[p->thread_id];
4200         struct instruction *ip = t->ip;
4201
4202         /* Structs. */
4203         __instr_regwr_rhh_exec(p, t, ip);
4204
4205         /* Thread. */
4206         thread_ip_inc(p);
4207 }
4208
4209 static inline void
4210 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4211 {
4212         struct thread *t = &p->threads[p->thread_id];
4213         struct instruction *ip = t->ip;
4214
4215         /* Structs. */
4216         __instr_regwr_rhm_exec(p, t, ip);
4217
4218         /* Thread. */
4219         thread_ip_inc(p);
4220 }
4221
4222 static inline void
4223 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4224 {
4225         struct thread *t = &p->threads[p->thread_id];
4226         struct instruction *ip = t->ip;
4227
4228         /* Structs. */
4229         __instr_regwr_rmh_exec(p, t, ip);
4230
4231         /* Thread. */
4232         thread_ip_inc(p);
4233 }
4234
4235 static inline void
4236 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4237 {
4238         struct thread *t = &p->threads[p->thread_id];
4239         struct instruction *ip = t->ip;
4240
4241         /* Structs. */
4242         __instr_regwr_rmm_exec(p, t, ip);
4243
4244         /* Thread. */
4245         thread_ip_inc(p);
4246 }
4247
4248 static inline void
4249 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4250 {
4251         struct thread *t = &p->threads[p->thread_id];
4252         struct instruction *ip = t->ip;
4253
4254         /* Structs. */
4255         __instr_regwr_rhi_exec(p, t, ip);
4256
4257         /* Thread. */
4258         thread_ip_inc(p);
4259 }
4260
4261 static inline void
4262 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4263 {
4264         struct thread *t = &p->threads[p->thread_id];
4265         struct instruction *ip = t->ip;
4266
4267         /* Structs. */
4268         __instr_regwr_rmi_exec(p, t, ip);
4269
4270         /* Thread. */
4271         thread_ip_inc(p);
4272 }
4273
4274 static inline void
4275 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4276 {
4277         struct thread *t = &p->threads[p->thread_id];
4278         struct instruction *ip = t->ip;
4279
4280         /* Structs. */
4281         __instr_regwr_rih_exec(p, t, ip);
4282
4283         /* Thread. */
4284         thread_ip_inc(p);
4285 }
4286
4287 static inline void
4288 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4289 {
4290         struct thread *t = &p->threads[p->thread_id];
4291         struct instruction *ip = t->ip;
4292
4293         /* Structs. */
4294         __instr_regwr_rim_exec(p, t, ip);
4295
4296         /* Thread. */
4297         thread_ip_inc(p);
4298 }
4299
4300 static inline void
4301 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4302 {
4303         struct thread *t = &p->threads[p->thread_id];
4304         struct instruction *ip = t->ip;
4305
4306         /* Structs. */
4307         __instr_regwr_rii_exec(p, t, ip);
4308
4309         /* Thread. */
4310         thread_ip_inc(p);
4311 }
4312
4313 static inline void
4314 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4315 {
4316         struct thread *t = &p->threads[p->thread_id];
4317         struct instruction *ip = t->ip;
4318
4319         /* Structs. */
4320         __instr_regadd_rhh_exec(p, t, ip);
4321
4322         /* Thread. */
4323         thread_ip_inc(p);
4324 }
4325
4326 static inline void
4327 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4328 {
4329         struct thread *t = &p->threads[p->thread_id];
4330         struct instruction *ip = t->ip;
4331
4332         /* Structs. */
4333         __instr_regadd_rhm_exec(p, t, ip);
4334
4335         /* Thread. */
4336         thread_ip_inc(p);
4337 }
4338
4339 static inline void
4340 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4341 {
4342         struct thread *t = &p->threads[p->thread_id];
4343         struct instruction *ip = t->ip;
4344
4345         /* Structs. */
4346         __instr_regadd_rmh_exec(p, t, ip);
4347
4348         /* Thread. */
4349         thread_ip_inc(p);
4350 }
4351
4352 static inline void
4353 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4354 {
4355         struct thread *t = &p->threads[p->thread_id];
4356         struct instruction *ip = t->ip;
4357
4358         /* Structs. */
4359         __instr_regadd_rmm_exec(p, t, ip);
4360
4361         /* Thread. */
4362         thread_ip_inc(p);
4363 }
4364
4365 static inline void
4366 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4367 {
4368         struct thread *t = &p->threads[p->thread_id];
4369         struct instruction *ip = t->ip;
4370
4371         /* Structs. */
4372         __instr_regadd_rhi_exec(p, t, ip);
4373
4374         /* Thread. */
4375         thread_ip_inc(p);
4376 }
4377
4378 static inline void
4379 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4380 {
4381         struct thread *t = &p->threads[p->thread_id];
4382         struct instruction *ip = t->ip;
4383
4384         /* Structs. */
4385         __instr_regadd_rmi_exec(p, t, ip);
4386
4387         /* Thread. */
4388         thread_ip_inc(p);
4389 }
4390
4391 static inline void
4392 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4393 {
4394         struct thread *t = &p->threads[p->thread_id];
4395         struct instruction *ip = t->ip;
4396
4397         /* Structs. */
4398         __instr_regadd_rih_exec(p, t, ip);
4399
4400         /* Thread. */
4401         thread_ip_inc(p);
4402 }
4403
4404 static inline void
4405 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4406 {
4407         struct thread *t = &p->threads[p->thread_id];
4408         struct instruction *ip = t->ip;
4409
4410         /* Structs. */
4411         __instr_regadd_rim_exec(p, t, ip);
4412
4413         /* Thread. */
4414         thread_ip_inc(p);
4415 }
4416
4417 static inline void
4418 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4419 {
4420         struct thread *t = &p->threads[p->thread_id];
4421         struct instruction *ip = t->ip;
4422
4423         /* Structs. */
4424         __instr_regadd_rii_exec(p, t, ip);
4425
4426         /* Thread. */
4427         thread_ip_inc(p);
4428 }
4429
4430 /*
4431  * metarray.
4432  */
4433 static struct metarray *
4434 metarray_find(struct rte_swx_pipeline *p, const char *name);
4435
4436 static int
4437 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4438                             struct action *action,
4439                             char **tokens,
4440                             int n_tokens,
4441                             struct instruction *instr,
4442                             struct instruction_data *data __rte_unused)
4443 {
4444         char *metarray = tokens[1], *idx = tokens[2];
4445         struct metarray *m;
4446         struct field *fidx;
4447         uint32_t idx_struct_id, idx_val;
4448
4449         CHECK(n_tokens == 3, EINVAL);
4450
4451         m = metarray_find(p, metarray);
4452         CHECK(m, EINVAL);
4453
4454         /* METPREFETCH_H, METPREFETCH_M. */
4455         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4456         if (fidx) {
4457                 CHECK(!fidx->var_size, EINVAL);
4458
4459                 instr->type = INSTR_METPREFETCH_M;
4460                 if (idx[0] == 'h')
4461                         instr->type = INSTR_METPREFETCH_H;
4462
4463                 instr->meter.metarray_id = m->id;
4464                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4465                 instr->meter.idx.n_bits = fidx->n_bits;
4466                 instr->meter.idx.offset = fidx->offset / 8;
4467                 return 0;
4468         }
4469
4470         /* METPREFETCH_I. */
4471         idx_val = strtoul(idx, &idx, 0);
4472         CHECK(!idx[0], EINVAL);
4473
4474         instr->type = INSTR_METPREFETCH_I;
4475         instr->meter.metarray_id = m->id;
4476         instr->meter.idx_val = idx_val;
4477         return 0;
4478 }
4479
4480 static int
4481 instr_meter_translate(struct rte_swx_pipeline *p,
4482                       struct action *action,
4483                       char **tokens,
4484                       int n_tokens,
4485                       struct instruction *instr,
4486                       struct instruction_data *data __rte_unused)
4487 {
4488         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4489         char *color_in = tokens[4], *color_out = tokens[5];
4490         struct metarray *m;
4491         struct field *fidx, *flength, *fcin, *fcout;
4492         uint32_t idx_struct_id, length_struct_id;
4493         uint32_t color_in_struct_id, color_out_struct_id;
4494
4495         CHECK(n_tokens == 6, EINVAL);
4496
4497         m = metarray_find(p, metarray);
4498         CHECK(m, EINVAL);
4499
4500         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4501
4502         flength = struct_field_parse(p, action, length, &length_struct_id);
4503         CHECK(flength, EINVAL);
4504         CHECK(!flength->var_size, EINVAL);
4505
4506         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4507
4508         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4509         CHECK(fcout, EINVAL);
4510         CHECK(!fcout->var_size, EINVAL);
4511
4512         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4513         if (fidx && fcin) {
4514                 CHECK(!fidx->var_size, EINVAL);
4515                 CHECK(!fcin->var_size, EINVAL);
4516
4517                 instr->type = INSTR_METER_MMM;
4518                 if (idx[0] == 'h' && length[0] == 'h')
4519                         instr->type = INSTR_METER_HHM;
4520                 if (idx[0] == 'h' && length[0] != 'h')
4521                         instr->type = INSTR_METER_HMM;
4522                 if (idx[0] != 'h' && length[0] == 'h')
4523                         instr->type = INSTR_METER_MHM;
4524
4525                 instr->meter.metarray_id = m->id;
4526
4527                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4528                 instr->meter.idx.n_bits = fidx->n_bits;
4529                 instr->meter.idx.offset = fidx->offset / 8;
4530
4531                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4532                 instr->meter.length.n_bits = flength->n_bits;
4533                 instr->meter.length.offset = flength->offset / 8;
4534
4535                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4536                 instr->meter.color_in.n_bits = fcin->n_bits;
4537                 instr->meter.color_in.offset = fcin->offset / 8;
4538
4539                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4540                 instr->meter.color_out.n_bits = fcout->n_bits;
4541                 instr->meter.color_out.offset = fcout->offset / 8;
4542
4543                 return 0;
4544         }
4545
4546         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4547         if (fidx && !fcin) {
4548                 uint32_t color_in_val;
4549
4550                 CHECK(!fidx->var_size, EINVAL);
4551
4552                 color_in_val = strtoul(color_in, &color_in, 0);
4553                 CHECK(!color_in[0], EINVAL);
4554
4555                 instr->type = INSTR_METER_MMI;
4556                 if (idx[0] == 'h' && length[0] == 'h')
4557                         instr->type = INSTR_METER_HHI;
4558                 if (idx[0] == 'h' && length[0] != 'h')
4559                         instr->type = INSTR_METER_HMI;
4560                 if (idx[0] != 'h' && length[0] == 'h')
4561                         instr->type = INSTR_METER_MHI;
4562
4563                 instr->meter.metarray_id = m->id;
4564
4565                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4566                 instr->meter.idx.n_bits = fidx->n_bits;
4567                 instr->meter.idx.offset = fidx->offset / 8;
4568
4569                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4570                 instr->meter.length.n_bits = flength->n_bits;
4571                 instr->meter.length.offset = flength->offset / 8;
4572
4573                 instr->meter.color_in_val = color_in_val;
4574
4575                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4576                 instr->meter.color_out.n_bits = fcout->n_bits;
4577                 instr->meter.color_out.offset = fcout->offset / 8;
4578
4579                 return 0;
4580         }
4581
4582         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4583         if (!fidx && fcin) {
4584                 uint32_t idx_val;
4585
4586                 idx_val = strtoul(idx, &idx, 0);
4587                 CHECK(!idx[0], EINVAL);
4588
4589                 CHECK(!fcin->var_size, EINVAL);
4590
4591                 instr->type = INSTR_METER_IMM;
4592                 if (length[0] == 'h')
4593                         instr->type = INSTR_METER_IHM;
4594
4595                 instr->meter.metarray_id = m->id;
4596
4597                 instr->meter.idx_val = idx_val;
4598
4599                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4600                 instr->meter.length.n_bits = flength->n_bits;
4601                 instr->meter.length.offset = flength->offset / 8;
4602
4603                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4604                 instr->meter.color_in.n_bits = fcin->n_bits;
4605                 instr->meter.color_in.offset = fcin->offset / 8;
4606
4607                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4608                 instr->meter.color_out.n_bits = fcout->n_bits;
4609                 instr->meter.color_out.offset = fcout->offset / 8;
4610
4611                 return 0;
4612         }
4613
4614         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4615         if (!fidx && !fcin) {
4616                 uint32_t idx_val, color_in_val;
4617
4618                 idx_val = strtoul(idx, &idx, 0);
4619                 CHECK(!idx[0], EINVAL);
4620
4621                 color_in_val = strtoul(color_in, &color_in, 0);
4622                 CHECK(!color_in[0], EINVAL);
4623
4624                 instr->type = INSTR_METER_IMI;
4625                 if (length[0] == 'h')
4626                         instr->type = INSTR_METER_IHI;
4627
4628                 instr->meter.metarray_id = m->id;
4629
4630                 instr->meter.idx_val = idx_val;
4631
4632                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4633                 instr->meter.length.n_bits = flength->n_bits;
4634                 instr->meter.length.offset = flength->offset / 8;
4635
4636                 instr->meter.color_in_val = color_in_val;
4637
4638                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4639                 instr->meter.color_out.n_bits = fcout->n_bits;
4640                 instr->meter.color_out.offset = fcout->offset / 8;
4641
4642                 return 0;
4643         }
4644
4645         CHECK(0, EINVAL);
4646 }
4647
4648 static inline void
4649 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4650 {
4651         struct thread *t = &p->threads[p->thread_id];
4652         struct instruction *ip = t->ip;
4653
4654         /* Structs. */
4655         __instr_metprefetch_h_exec(p, t, ip);
4656
4657         /* Thread. */
4658         thread_ip_inc(p);
4659 }
4660
4661 static inline void
4662 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4663 {
4664         struct thread *t = &p->threads[p->thread_id];
4665         struct instruction *ip = t->ip;
4666
4667         /* Structs. */
4668         __instr_metprefetch_m_exec(p, t, ip);
4669
4670         /* Thread. */
4671         thread_ip_inc(p);
4672 }
4673
4674 static inline void
4675 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4676 {
4677         struct thread *t = &p->threads[p->thread_id];
4678         struct instruction *ip = t->ip;
4679
4680         /* Structs. */
4681         __instr_metprefetch_i_exec(p, t, ip);
4682
4683         /* Thread. */
4684         thread_ip_inc(p);
4685 }
4686
4687 static inline void
4688 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4689 {
4690         struct thread *t = &p->threads[p->thread_id];
4691         struct instruction *ip = t->ip;
4692
4693         /* Structs. */
4694         __instr_meter_hhm_exec(p, t, ip);
4695
4696         /* Thread. */
4697         thread_ip_inc(p);
4698 }
4699
4700 static inline void
4701 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4702 {
4703         struct thread *t = &p->threads[p->thread_id];
4704         struct instruction *ip = t->ip;
4705
4706         /* Structs. */
4707         __instr_meter_hhi_exec(p, t, ip);
4708
4709         /* Thread. */
4710         thread_ip_inc(p);
4711 }
4712
4713 static inline void
4714 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4715 {
4716         struct thread *t = &p->threads[p->thread_id];
4717         struct instruction *ip = t->ip;
4718
4719         /* Structs. */
4720         __instr_meter_hmm_exec(p, t, ip);
4721
4722         /* Thread. */
4723         thread_ip_inc(p);
4724 }
4725
4726 static inline void
4727 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4728 {
4729         struct thread *t = &p->threads[p->thread_id];
4730         struct instruction *ip = t->ip;
4731
4732         /* Structs. */
4733         __instr_meter_hmi_exec(p, t, ip);
4734
4735         /* Thread. */
4736         thread_ip_inc(p);
4737 }
4738
4739 static inline void
4740 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4741 {
4742         struct thread *t = &p->threads[p->thread_id];
4743         struct instruction *ip = t->ip;
4744
4745         /* Structs. */
4746         __instr_meter_mhm_exec(p, t, ip);
4747
4748         /* Thread. */
4749         thread_ip_inc(p);
4750 }
4751
4752 static inline void
4753 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4754 {
4755         struct thread *t = &p->threads[p->thread_id];
4756         struct instruction *ip = t->ip;
4757
4758         /* Structs. */
4759         __instr_meter_mhi_exec(p, t, ip);
4760
4761         /* Thread. */
4762         thread_ip_inc(p);
4763 }
4764
4765 static inline void
4766 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4767 {
4768         struct thread *t = &p->threads[p->thread_id];
4769         struct instruction *ip = t->ip;
4770
4771         /* Structs. */
4772         __instr_meter_mmm_exec(p, t, ip);
4773
4774         /* Thread. */
4775         thread_ip_inc(p);
4776 }
4777
4778 static inline void
4779 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4780 {
4781         struct thread *t = &p->threads[p->thread_id];
4782         struct instruction *ip = t->ip;
4783
4784         /* Structs. */
4785         __instr_meter_mmi_exec(p, t, ip);
4786
4787         /* Thread. */
4788         thread_ip_inc(p);
4789 }
4790
4791 static inline void
4792 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4793 {
4794         struct thread *t = &p->threads[p->thread_id];
4795         struct instruction *ip = t->ip;
4796
4797         /* Structs. */
4798         __instr_meter_ihm_exec(p, t, ip);
4799
4800         /* Thread. */
4801         thread_ip_inc(p);
4802 }
4803
4804 static inline void
4805 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4806 {
4807         struct thread *t = &p->threads[p->thread_id];
4808         struct instruction *ip = t->ip;
4809
4810         /* Structs. */
4811         __instr_meter_ihi_exec(p, t, ip);
4812
4813         /* Thread. */
4814         thread_ip_inc(p);
4815 }
4816
4817 static inline void
4818 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4819 {
4820         struct thread *t = &p->threads[p->thread_id];
4821         struct instruction *ip = t->ip;
4822
4823         /* Structs. */
4824         __instr_meter_imm_exec(p, t, ip);
4825
4826         /* Thread. */
4827         thread_ip_inc(p);
4828 }
4829
4830 static inline void
4831 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4832 {
4833         struct thread *t = &p->threads[p->thread_id];
4834         struct instruction *ip = t->ip;
4835
4836         /* Structs. */
4837         __instr_meter_imi_exec(p, t, ip);
4838
4839         /* Thread. */
4840         thread_ip_inc(p);
4841 }
4842
4843 /*
4844  * jmp.
4845  */
4846 static int
4847 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4848                     struct action *action __rte_unused,
4849                     char **tokens,
4850                     int n_tokens,
4851                     struct instruction *instr,
4852                     struct instruction_data *data)
4853 {
4854         CHECK(n_tokens == 2, EINVAL);
4855
4856         strcpy(data->jmp_label, tokens[1]);
4857
4858         instr->type = INSTR_JMP;
4859         instr->jmp.ip = NULL; /* Resolved later. */
4860         return 0;
4861 }
4862
4863 static int
4864 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4865                           struct action *action __rte_unused,
4866                           char **tokens,
4867                           int n_tokens,
4868                           struct instruction *instr,
4869                           struct instruction_data *data)
4870 {
4871         struct header *h;
4872
4873         CHECK(n_tokens == 3, EINVAL);
4874
4875         strcpy(data->jmp_label, tokens[1]);
4876
4877         h = header_parse(p, tokens[2]);
4878         CHECK(h, EINVAL);
4879
4880         instr->type = INSTR_JMP_VALID;
4881         instr->jmp.ip = NULL; /* Resolved later. */
4882         instr->jmp.header_id = h->id;
4883         return 0;
4884 }
4885
4886 static int
4887 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4888                             struct action *action __rte_unused,
4889                             char **tokens,
4890                             int n_tokens,
4891                             struct instruction *instr,
4892                             struct instruction_data *data)
4893 {
4894         struct header *h;
4895
4896         CHECK(n_tokens == 3, EINVAL);
4897
4898         strcpy(data->jmp_label, tokens[1]);
4899
4900         h = header_parse(p, tokens[2]);
4901         CHECK(h, EINVAL);
4902
4903         instr->type = INSTR_JMP_INVALID;
4904         instr->jmp.ip = NULL; /* Resolved later. */
4905         instr->jmp.header_id = h->id;
4906         return 0;
4907 }
4908
4909 static int
4910 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4911                         struct action *action,
4912                         char **tokens,
4913                         int n_tokens,
4914                         struct instruction *instr,
4915                         struct instruction_data *data)
4916 {
4917         CHECK(!action, EINVAL);
4918         CHECK(n_tokens == 2, EINVAL);
4919
4920         strcpy(data->jmp_label, tokens[1]);
4921
4922         instr->type = INSTR_JMP_HIT;
4923         instr->jmp.ip = NULL; /* Resolved later. */
4924         return 0;
4925 }
4926
4927 static int
4928 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4929                          struct action *action,
4930                          char **tokens,
4931                          int n_tokens,
4932                          struct instruction *instr,
4933                          struct instruction_data *data)
4934 {
4935         CHECK(!action, EINVAL);
4936         CHECK(n_tokens == 2, EINVAL);
4937
4938         strcpy(data->jmp_label, tokens[1]);
4939
4940         instr->type = INSTR_JMP_MISS;
4941         instr->jmp.ip = NULL; /* Resolved later. */
4942         return 0;
4943 }
4944
4945 static int
4946 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4947                                struct action *action,
4948                                char **tokens,
4949                                int n_tokens,
4950                                struct instruction *instr,
4951                                struct instruction_data *data)
4952 {
4953         struct action *a;
4954
4955         CHECK(!action, EINVAL);
4956         CHECK(n_tokens == 3, EINVAL);
4957
4958         strcpy(data->jmp_label, tokens[1]);
4959
4960         a = action_find(p, tokens[2]);
4961         CHECK(a, EINVAL);
4962
4963         instr->type = INSTR_JMP_ACTION_HIT;
4964         instr->jmp.ip = NULL; /* Resolved later. */
4965         instr->jmp.action_id = a->id;
4966         return 0;
4967 }
4968
4969 static int
4970 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4971                                 struct action *action,
4972                                 char **tokens,
4973                                 int n_tokens,
4974                                 struct instruction *instr,
4975                                 struct instruction_data *data)
4976 {
4977         struct action *a;
4978
4979         CHECK(!action, EINVAL);
4980         CHECK(n_tokens == 3, EINVAL);
4981
4982         strcpy(data->jmp_label, tokens[1]);
4983
4984         a = action_find(p, tokens[2]);
4985         CHECK(a, EINVAL);
4986
4987         instr->type = INSTR_JMP_ACTION_MISS;
4988         instr->jmp.ip = NULL; /* Resolved later. */
4989         instr->jmp.action_id = a->id;
4990         return 0;
4991 }
4992
4993 static int
4994 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4995                        struct action *action,
4996                        char **tokens,
4997                        int n_tokens,
4998                        struct instruction *instr,
4999                        struct instruction_data *data)
5000 {
5001         char *a = tokens[2], *b = tokens[3];
5002         struct field *fa, *fb;
5003         uint64_t b_val;
5004         uint32_t a_struct_id, b_struct_id;
5005
5006         CHECK(n_tokens == 4, EINVAL);
5007
5008         strcpy(data->jmp_label, tokens[1]);
5009
5010         fa = struct_field_parse(p, action, a, &a_struct_id);
5011         CHECK(fa, EINVAL);
5012         CHECK(!fa->var_size, EINVAL);
5013
5014         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5015         fb = struct_field_parse(p, action, b, &b_struct_id);
5016         if (fb) {
5017                 CHECK(!fb->var_size, EINVAL);
5018
5019                 instr->type = INSTR_JMP_EQ;
5020                 if (a[0] != 'h' && b[0] == 'h')
5021                         instr->type = INSTR_JMP_EQ_MH;
5022                 if (a[0] == 'h' && b[0] != 'h')
5023                         instr->type = INSTR_JMP_EQ_HM;
5024                 if (a[0] == 'h' && b[0] == 'h')
5025                         instr->type = INSTR_JMP_EQ_HH;
5026                 instr->jmp.ip = NULL; /* Resolved later. */
5027
5028                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5029                 instr->jmp.a.n_bits = fa->n_bits;
5030                 instr->jmp.a.offset = fa->offset / 8;
5031                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5032                 instr->jmp.b.n_bits = fb->n_bits;
5033                 instr->jmp.b.offset = fb->offset / 8;
5034                 return 0;
5035         }
5036
5037         /* JMP_EQ_I. */
5038         b_val = strtoull(b, &b, 0);
5039         CHECK(!b[0], EINVAL);
5040
5041         if (a[0] == 'h')
5042                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5043
5044         instr->type = INSTR_JMP_EQ_I;
5045         instr->jmp.ip = NULL; /* Resolved later. */
5046         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5047         instr->jmp.a.n_bits = fa->n_bits;
5048         instr->jmp.a.offset = fa->offset / 8;
5049         instr->jmp.b_val = b_val;
5050         return 0;
5051 }
5052
5053 static int
5054 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5055                         struct action *action,
5056                         char **tokens,
5057                         int n_tokens,
5058                         struct instruction *instr,
5059                         struct instruction_data *data)
5060 {
5061         char *a = tokens[2], *b = tokens[3];
5062         struct field *fa, *fb;
5063         uint64_t b_val;
5064         uint32_t a_struct_id, b_struct_id;
5065
5066         CHECK(n_tokens == 4, EINVAL);
5067
5068         strcpy(data->jmp_label, tokens[1]);
5069
5070         fa = struct_field_parse(p, action, a, &a_struct_id);
5071         CHECK(fa, EINVAL);
5072         CHECK(!fa->var_size, EINVAL);
5073
5074         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5075         fb = struct_field_parse(p, action, b, &b_struct_id);
5076         if (fb) {
5077                 CHECK(!fb->var_size, EINVAL);
5078
5079                 instr->type = INSTR_JMP_NEQ;
5080                 if (a[0] != 'h' && b[0] == 'h')
5081                         instr->type = INSTR_JMP_NEQ_MH;
5082                 if (a[0] == 'h' && b[0] != 'h')
5083                         instr->type = INSTR_JMP_NEQ_HM;
5084                 if (a[0] == 'h' && b[0] == 'h')
5085                         instr->type = INSTR_JMP_NEQ_HH;
5086                 instr->jmp.ip = NULL; /* Resolved later. */
5087
5088                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5089                 instr->jmp.a.n_bits = fa->n_bits;
5090                 instr->jmp.a.offset = fa->offset / 8;
5091                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5092                 instr->jmp.b.n_bits = fb->n_bits;
5093                 instr->jmp.b.offset = fb->offset / 8;
5094                 return 0;
5095         }
5096
5097         /* JMP_NEQ_I. */
5098         b_val = strtoull(b, &b, 0);
5099         CHECK(!b[0], EINVAL);
5100
5101         if (a[0] == 'h')
5102                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5103
5104         instr->type = INSTR_JMP_NEQ_I;
5105         instr->jmp.ip = NULL; /* Resolved later. */
5106         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5107         instr->jmp.a.n_bits = fa->n_bits;
5108         instr->jmp.a.offset = fa->offset / 8;
5109         instr->jmp.b_val = b_val;
5110         return 0;
5111 }
5112
5113 static int
5114 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5115                        struct action *action,
5116                        char **tokens,
5117                        int n_tokens,
5118                        struct instruction *instr,
5119                        struct instruction_data *data)
5120 {
5121         char *a = tokens[2], *b = tokens[3];
5122         struct field *fa, *fb;
5123         uint64_t b_val;
5124         uint32_t a_struct_id, b_struct_id;
5125
5126         CHECK(n_tokens == 4, EINVAL);
5127
5128         strcpy(data->jmp_label, tokens[1]);
5129
5130         fa = struct_field_parse(p, action, a, &a_struct_id);
5131         CHECK(fa, EINVAL);
5132         CHECK(!fa->var_size, EINVAL);
5133
5134         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5135         fb = struct_field_parse(p, action, b, &b_struct_id);
5136         if (fb) {
5137                 CHECK(!fb->var_size, EINVAL);
5138
5139                 instr->type = INSTR_JMP_LT;
5140                 if (a[0] == 'h' && b[0] != 'h')
5141                         instr->type = INSTR_JMP_LT_HM;
5142                 if (a[0] != 'h' && b[0] == 'h')
5143                         instr->type = INSTR_JMP_LT_MH;
5144                 if (a[0] == 'h' && b[0] == 'h')
5145                         instr->type = INSTR_JMP_LT_HH;
5146                 instr->jmp.ip = NULL; /* Resolved later. */
5147
5148                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5149                 instr->jmp.a.n_bits = fa->n_bits;
5150                 instr->jmp.a.offset = fa->offset / 8;
5151                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5152                 instr->jmp.b.n_bits = fb->n_bits;
5153                 instr->jmp.b.offset = fb->offset / 8;
5154                 return 0;
5155         }
5156
5157         /* JMP_LT_MI, JMP_LT_HI. */
5158         b_val = strtoull(b, &b, 0);
5159         CHECK(!b[0], EINVAL);
5160
5161         instr->type = INSTR_JMP_LT_MI;
5162         if (a[0] == 'h')
5163                 instr->type = INSTR_JMP_LT_HI;
5164         instr->jmp.ip = NULL; /* Resolved later. */
5165
5166         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5167         instr->jmp.a.n_bits = fa->n_bits;
5168         instr->jmp.a.offset = fa->offset / 8;
5169         instr->jmp.b_val = b_val;
5170         return 0;
5171 }
5172
5173 static int
5174 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5175                        struct action *action,
5176                        char **tokens,
5177                        int n_tokens,
5178                        struct instruction *instr,
5179                        struct instruction_data *data)
5180 {
5181         char *a = tokens[2], *b = tokens[3];
5182         struct field *fa, *fb;
5183         uint64_t b_val;
5184         uint32_t a_struct_id, b_struct_id;
5185
5186         CHECK(n_tokens == 4, EINVAL);
5187
5188         strcpy(data->jmp_label, tokens[1]);
5189
5190         fa = struct_field_parse(p, action, a, &a_struct_id);
5191         CHECK(fa, EINVAL);
5192         CHECK(!fa->var_size, EINVAL);
5193
5194         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5195         fb = struct_field_parse(p, action, b, &b_struct_id);
5196         if (fb) {
5197                 CHECK(!fb->var_size, EINVAL);
5198
5199                 instr->type = INSTR_JMP_GT;
5200                 if (a[0] == 'h' && b[0] != 'h')
5201                         instr->type = INSTR_JMP_GT_HM;
5202                 if (a[0] != 'h' && b[0] == 'h')
5203                         instr->type = INSTR_JMP_GT_MH;
5204                 if (a[0] == 'h' && b[0] == 'h')
5205                         instr->type = INSTR_JMP_GT_HH;
5206                 instr->jmp.ip = NULL; /* Resolved later. */
5207
5208                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5209                 instr->jmp.a.n_bits = fa->n_bits;
5210                 instr->jmp.a.offset = fa->offset / 8;
5211                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5212                 instr->jmp.b.n_bits = fb->n_bits;
5213                 instr->jmp.b.offset = fb->offset / 8;
5214                 return 0;
5215         }
5216
5217         /* JMP_GT_MI, JMP_GT_HI. */
5218         b_val = strtoull(b, &b, 0);
5219         CHECK(!b[0], EINVAL);
5220
5221         instr->type = INSTR_JMP_GT_MI;
5222         if (a[0] == 'h')
5223                 instr->type = INSTR_JMP_GT_HI;
5224         instr->jmp.ip = NULL; /* Resolved later. */
5225
5226         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5227         instr->jmp.a.n_bits = fa->n_bits;
5228         instr->jmp.a.offset = fa->offset / 8;
5229         instr->jmp.b_val = b_val;
5230         return 0;
5231 }
5232
5233 static inline void
5234 instr_jmp_exec(struct rte_swx_pipeline *p)
5235 {
5236         struct thread *t = &p->threads[p->thread_id];
5237         struct instruction *ip = t->ip;
5238
5239         TRACE("[Thread %2u] jmp\n", p->thread_id);
5240
5241         thread_ip_set(t, ip->jmp.ip);
5242 }
5243
5244 static inline void
5245 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5246 {
5247         struct thread *t = &p->threads[p->thread_id];
5248         struct instruction *ip = t->ip;
5249         uint32_t header_id = ip->jmp.header_id;
5250
5251         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5252
5253         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5254 }
5255
5256 static inline void
5257 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5258 {
5259         struct thread *t = &p->threads[p->thread_id];
5260         struct instruction *ip = t->ip;
5261         uint32_t header_id = ip->jmp.header_id;
5262
5263         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5264
5265         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5266 }
5267
5268 static inline void
5269 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5270 {
5271         struct thread *t = &p->threads[p->thread_id];
5272         struct instruction *ip = t->ip;
5273         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5274
5275         TRACE("[Thread %2u] jmph\n", p->thread_id);
5276
5277         t->ip = ip_next[t->hit];
5278 }
5279
5280 static inline void
5281 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5282 {
5283         struct thread *t = &p->threads[p->thread_id];
5284         struct instruction *ip = t->ip;
5285         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5286
5287         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5288
5289         t->ip = ip_next[t->hit];
5290 }
5291
5292 static inline void
5293 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5294 {
5295         struct thread *t = &p->threads[p->thread_id];
5296         struct instruction *ip = t->ip;
5297
5298         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5299
5300         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5301 }
5302
5303 static inline void
5304 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5305 {
5306         struct thread *t = &p->threads[p->thread_id];
5307         struct instruction *ip = t->ip;
5308
5309         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5310
5311         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5312 }
5313
5314 static inline void
5315 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5316 {
5317         struct thread *t = &p->threads[p->thread_id];
5318         struct instruction *ip = t->ip;
5319
5320         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5321
5322         JMP_CMP(t, ip, ==);
5323 }
5324
5325 static inline void
5326 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5327 {
5328         struct thread *t = &p->threads[p->thread_id];
5329         struct instruction *ip = t->ip;
5330
5331         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5332
5333         JMP_CMP_MH(t, ip, ==);
5334 }
5335
5336 static inline void
5337 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5338 {
5339         struct thread *t = &p->threads[p->thread_id];
5340         struct instruction *ip = t->ip;
5341
5342         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5343
5344         JMP_CMP_HM(t, ip, ==);
5345 }
5346
5347 static inline void
5348 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5349 {
5350         struct thread *t = &p->threads[p->thread_id];
5351         struct instruction *ip = t->ip;
5352
5353         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5354
5355         JMP_CMP_HH_FAST(t, ip, ==);
5356 }
5357
5358 static inline void
5359 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5360 {
5361         struct thread *t = &p->threads[p->thread_id];
5362         struct instruction *ip = t->ip;
5363
5364         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5365
5366         JMP_CMP_I(t, ip, ==);
5367 }
5368
5369 static inline void
5370 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5371 {
5372         struct thread *t = &p->threads[p->thread_id];
5373         struct instruction *ip = t->ip;
5374
5375         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5376
5377         JMP_CMP(t, ip, !=);
5378 }
5379
5380 static inline void
5381 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5382 {
5383         struct thread *t = &p->threads[p->thread_id];
5384         struct instruction *ip = t->ip;
5385
5386         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5387
5388         JMP_CMP_MH(t, ip, !=);
5389 }
5390
5391 static inline void
5392 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5393 {
5394         struct thread *t = &p->threads[p->thread_id];
5395         struct instruction *ip = t->ip;
5396
5397         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5398
5399         JMP_CMP_HM(t, ip, !=);
5400 }
5401
5402 static inline void
5403 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5404 {
5405         struct thread *t = &p->threads[p->thread_id];
5406         struct instruction *ip = t->ip;
5407
5408         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5409
5410         JMP_CMP_HH_FAST(t, ip, !=);
5411 }
5412
5413 static inline void
5414 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5415 {
5416         struct thread *t = &p->threads[p->thread_id];
5417         struct instruction *ip = t->ip;
5418
5419         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5420
5421         JMP_CMP_I(t, ip, !=);
5422 }
5423
5424 static inline void
5425 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5426 {
5427         struct thread *t = &p->threads[p->thread_id];
5428         struct instruction *ip = t->ip;
5429
5430         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5431
5432         JMP_CMP(t, ip, <);
5433 }
5434
5435 static inline void
5436 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5437 {
5438         struct thread *t = &p->threads[p->thread_id];
5439         struct instruction *ip = t->ip;
5440
5441         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5442
5443         JMP_CMP_MH(t, ip, <);
5444 }
5445
5446 static inline void
5447 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5448 {
5449         struct thread *t = &p->threads[p->thread_id];
5450         struct instruction *ip = t->ip;
5451
5452         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5453
5454         JMP_CMP_HM(t, ip, <);
5455 }
5456
5457 static inline void
5458 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5459 {
5460         struct thread *t = &p->threads[p->thread_id];
5461         struct instruction *ip = t->ip;
5462
5463         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5464
5465         JMP_CMP_HH(t, ip, <);
5466 }
5467
5468 static inline void
5469 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5470 {
5471         struct thread *t = &p->threads[p->thread_id];
5472         struct instruction *ip = t->ip;
5473
5474         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5475
5476         JMP_CMP_MI(t, ip, <);
5477 }
5478
5479 static inline void
5480 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5481 {
5482         struct thread *t = &p->threads[p->thread_id];
5483         struct instruction *ip = t->ip;
5484
5485         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5486
5487         JMP_CMP_HI(t, ip, <);
5488 }
5489
5490 static inline void
5491 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5492 {
5493         struct thread *t = &p->threads[p->thread_id];
5494         struct instruction *ip = t->ip;
5495
5496         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5497
5498         JMP_CMP(t, ip, >);
5499 }
5500
5501 static inline void
5502 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5503 {
5504         struct thread *t = &p->threads[p->thread_id];
5505         struct instruction *ip = t->ip;
5506
5507         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5508
5509         JMP_CMP_MH(t, ip, >);
5510 }
5511
5512 static inline void
5513 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5514 {
5515         struct thread *t = &p->threads[p->thread_id];
5516         struct instruction *ip = t->ip;
5517
5518         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5519
5520         JMP_CMP_HM(t, ip, >);
5521 }
5522
5523 static inline void
5524 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5525 {
5526         struct thread *t = &p->threads[p->thread_id];
5527         struct instruction *ip = t->ip;
5528
5529         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5530
5531         JMP_CMP_HH(t, ip, >);
5532 }
5533
5534 static inline void
5535 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5536 {
5537         struct thread *t = &p->threads[p->thread_id];
5538         struct instruction *ip = t->ip;
5539
5540         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5541
5542         JMP_CMP_MI(t, ip, >);
5543 }
5544
5545 static inline void
5546 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5547 {
5548         struct thread *t = &p->threads[p->thread_id];
5549         struct instruction *ip = t->ip;
5550
5551         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5552
5553         JMP_CMP_HI(t, ip, >);
5554 }
5555
5556 /*
5557  * return.
5558  */
5559 static int
5560 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5561                        struct action *action,
5562                        char **tokens __rte_unused,
5563                        int n_tokens,
5564                        struct instruction *instr,
5565                        struct instruction_data *data __rte_unused)
5566 {
5567         CHECK(action, EINVAL);
5568         CHECK(n_tokens == 1, EINVAL);
5569
5570         instr->type = INSTR_RETURN;
5571         return 0;
5572 }
5573
5574 static inline void
5575 instr_return_exec(struct rte_swx_pipeline *p)
5576 {
5577         struct thread *t = &p->threads[p->thread_id];
5578
5579         TRACE("[Thread %2u] return\n", p->thread_id);
5580
5581         t->ip = t->ret;
5582 }
5583
5584 static int
5585 instr_translate(struct rte_swx_pipeline *p,
5586                 struct action *action,
5587                 char *string,
5588                 struct instruction *instr,
5589                 struct instruction_data *data)
5590 {
5591         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5592         int n_tokens = 0, tpos = 0;
5593
5594         /* Parse the instruction string into tokens. */
5595         for ( ; ; ) {
5596                 char *token;
5597
5598                 token = strtok_r(string, " \t\v", &string);
5599                 if (!token)
5600                         break;
5601
5602                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5603                 CHECK_NAME(token, EINVAL);
5604
5605                 tokens[n_tokens] = token;
5606                 n_tokens++;
5607         }
5608
5609         CHECK(n_tokens, EINVAL);
5610
5611         /* Handle the optional instruction label. */
5612         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5613                 strcpy(data->label, tokens[0]);
5614
5615                 tpos += 2;
5616                 CHECK(n_tokens - tpos, EINVAL);
5617         }
5618
5619         /* Identify the instruction type. */
5620         if (!strcmp(tokens[tpos], "rx"))
5621                 return instr_rx_translate(p,
5622                                           action,
5623                                           &tokens[tpos],
5624                                           n_tokens - tpos,
5625                                           instr,
5626                                           data);
5627
5628         if (!strcmp(tokens[tpos], "tx"))
5629                 return instr_tx_translate(p,
5630                                           action,
5631                                           &tokens[tpos],
5632                                           n_tokens - tpos,
5633                                           instr,
5634                                           data);
5635
5636         if (!strcmp(tokens[tpos], "drop"))
5637                 return instr_drop_translate(p,
5638                                             action,
5639                                             &tokens[tpos],
5640                                             n_tokens - tpos,
5641                                             instr,
5642                                             data);
5643
5644         if (!strcmp(tokens[tpos], "extract"))
5645                 return instr_hdr_extract_translate(p,
5646                                                    action,
5647                                                    &tokens[tpos],
5648                                                    n_tokens - tpos,
5649                                                    instr,
5650                                                    data);
5651
5652         if (!strcmp(tokens[tpos], "lookahead"))
5653                 return instr_hdr_lookahead_translate(p,
5654                                                      action,
5655                                                      &tokens[tpos],
5656                                                      n_tokens - tpos,
5657                                                      instr,
5658                                                      data);
5659
5660         if (!strcmp(tokens[tpos], "emit"))
5661                 return instr_hdr_emit_translate(p,
5662                                                 action,
5663                                                 &tokens[tpos],
5664                                                 n_tokens - tpos,
5665                                                 instr,
5666                                                 data);
5667
5668         if (!strcmp(tokens[tpos], "validate"))
5669                 return instr_hdr_validate_translate(p,
5670                                                     action,
5671                                                     &tokens[tpos],
5672                                                     n_tokens - tpos,
5673                                                     instr,
5674                                                     data);
5675
5676         if (!strcmp(tokens[tpos], "invalidate"))
5677                 return instr_hdr_invalidate_translate(p,
5678                                                       action,
5679                                                       &tokens[tpos],
5680                                                       n_tokens - tpos,
5681                                                       instr,
5682                                                       data);
5683
5684         if (!strcmp(tokens[tpos], "mov"))
5685                 return instr_mov_translate(p,
5686                                            action,
5687                                            &tokens[tpos],
5688                                            n_tokens - tpos,
5689                                            instr,
5690                                            data);
5691
5692         if (!strcmp(tokens[tpos], "add"))
5693                 return instr_alu_add_translate(p,
5694                                                action,
5695                                                &tokens[tpos],
5696                                                n_tokens - tpos,
5697                                                instr,
5698                                                data);
5699
5700         if (!strcmp(tokens[tpos], "sub"))
5701                 return instr_alu_sub_translate(p,
5702                                                action,
5703                                                &tokens[tpos],
5704                                                n_tokens - tpos,
5705                                                instr,
5706                                                data);
5707
5708         if (!strcmp(tokens[tpos], "ckadd"))
5709                 return instr_alu_ckadd_translate(p,
5710                                                  action,
5711                                                  &tokens[tpos],
5712                                                  n_tokens - tpos,
5713                                                  instr,
5714                                                  data);
5715
5716         if (!strcmp(tokens[tpos], "cksub"))
5717                 return instr_alu_cksub_translate(p,
5718                                                  action,
5719                                                  &tokens[tpos],
5720                                                  n_tokens - tpos,
5721                                                  instr,
5722                                                  data);
5723
5724         if (!strcmp(tokens[tpos], "and"))
5725                 return instr_alu_and_translate(p,
5726                                                action,
5727                                                &tokens[tpos],
5728                                                n_tokens - tpos,
5729                                                instr,
5730                                                data);
5731
5732         if (!strcmp(tokens[tpos], "or"))
5733                 return instr_alu_or_translate(p,
5734                                               action,
5735                                               &tokens[tpos],
5736                                               n_tokens - tpos,
5737                                               instr,
5738                                               data);
5739
5740         if (!strcmp(tokens[tpos], "xor"))
5741                 return instr_alu_xor_translate(p,
5742                                                action,
5743                                                &tokens[tpos],
5744                                                n_tokens - tpos,
5745                                                instr,
5746                                                data);
5747
5748         if (!strcmp(tokens[tpos], "shl"))
5749                 return instr_alu_shl_translate(p,
5750                                                action,
5751                                                &tokens[tpos],
5752                                                n_tokens - tpos,
5753                                                instr,
5754                                                data);
5755
5756         if (!strcmp(tokens[tpos], "shr"))
5757                 return instr_alu_shr_translate(p,
5758                                                action,
5759                                                &tokens[tpos],
5760                                                n_tokens - tpos,
5761                                                instr,
5762                                                data);
5763
5764         if (!strcmp(tokens[tpos], "regprefetch"))
5765                 return instr_regprefetch_translate(p,
5766                                                    action,
5767                                                    &tokens[tpos],
5768                                                    n_tokens - tpos,
5769                                                    instr,
5770                                                    data);
5771
5772         if (!strcmp(tokens[tpos], "regrd"))
5773                 return instr_regrd_translate(p,
5774                                              action,
5775                                              &tokens[tpos],
5776                                              n_tokens - tpos,
5777                                              instr,
5778                                              data);
5779
5780         if (!strcmp(tokens[tpos], "regwr"))
5781                 return instr_regwr_translate(p,
5782                                              action,
5783                                              &tokens[tpos],
5784                                              n_tokens - tpos,
5785                                              instr,
5786                                              data);
5787
5788         if (!strcmp(tokens[tpos], "regadd"))
5789                 return instr_regadd_translate(p,
5790                                               action,
5791                                               &tokens[tpos],
5792                                               n_tokens - tpos,
5793                                               instr,
5794                                               data);
5795
5796         if (!strcmp(tokens[tpos], "metprefetch"))
5797                 return instr_metprefetch_translate(p,
5798                                                    action,
5799                                                    &tokens[tpos],
5800                                                    n_tokens - tpos,
5801                                                    instr,
5802                                                    data);
5803
5804         if (!strcmp(tokens[tpos], "meter"))
5805                 return instr_meter_translate(p,
5806                                              action,
5807                                              &tokens[tpos],
5808                                              n_tokens - tpos,
5809                                              instr,
5810                                              data);
5811
5812         if (!strcmp(tokens[tpos], "table"))
5813                 return instr_table_translate(p,
5814                                              action,
5815                                              &tokens[tpos],
5816                                              n_tokens - tpos,
5817                                              instr,
5818                                              data);
5819
5820         if (!strcmp(tokens[tpos], "learn"))
5821                 return instr_learn_translate(p,
5822                                              action,
5823                                              &tokens[tpos],
5824                                              n_tokens - tpos,
5825                                              instr,
5826                                              data);
5827
5828         if (!strcmp(tokens[tpos], "forget"))
5829                 return instr_forget_translate(p,
5830                                               action,
5831                                               &tokens[tpos],
5832                                               n_tokens - tpos,
5833                                               instr,
5834                                               data);
5835
5836         if (!strcmp(tokens[tpos], "extern"))
5837                 return instr_extern_translate(p,
5838                                               action,
5839                                               &tokens[tpos],
5840                                               n_tokens - tpos,
5841                                               instr,
5842                                               data);
5843
5844         if (!strcmp(tokens[tpos], "jmp"))
5845                 return instr_jmp_translate(p,
5846                                            action,
5847                                            &tokens[tpos],
5848                                            n_tokens - tpos,
5849                                            instr,
5850                                            data);
5851
5852         if (!strcmp(tokens[tpos], "jmpv"))
5853                 return instr_jmp_valid_translate(p,
5854                                                  action,
5855                                                  &tokens[tpos],
5856                                                  n_tokens - tpos,
5857                                                  instr,
5858                                                  data);
5859
5860         if (!strcmp(tokens[tpos], "jmpnv"))
5861                 return instr_jmp_invalid_translate(p,
5862                                                    action,
5863                                                    &tokens[tpos],
5864                                                    n_tokens - tpos,
5865                                                    instr,
5866                                                    data);
5867
5868         if (!strcmp(tokens[tpos], "jmph"))
5869                 return instr_jmp_hit_translate(p,
5870                                                action,
5871                                                &tokens[tpos],
5872                                                n_tokens - tpos,
5873                                                instr,
5874                                                data);
5875
5876         if (!strcmp(tokens[tpos], "jmpnh"))
5877                 return instr_jmp_miss_translate(p,
5878                                                 action,
5879                                                 &tokens[tpos],
5880                                                 n_tokens - tpos,
5881                                                 instr,
5882                                                 data);
5883
5884         if (!strcmp(tokens[tpos], "jmpa"))
5885                 return instr_jmp_action_hit_translate(p,
5886                                                       action,
5887                                                       &tokens[tpos],
5888                                                       n_tokens - tpos,
5889                                                       instr,
5890                                                       data);
5891
5892         if (!strcmp(tokens[tpos], "jmpna"))
5893                 return instr_jmp_action_miss_translate(p,
5894                                                        action,
5895                                                        &tokens[tpos],
5896                                                        n_tokens - tpos,
5897                                                        instr,
5898                                                        data);
5899
5900         if (!strcmp(tokens[tpos], "jmpeq"))
5901                 return instr_jmp_eq_translate(p,
5902                                               action,
5903                                               &tokens[tpos],
5904                                               n_tokens - tpos,
5905                                               instr,
5906                                               data);
5907
5908         if (!strcmp(tokens[tpos], "jmpneq"))
5909                 return instr_jmp_neq_translate(p,
5910                                                action,
5911                                                &tokens[tpos],
5912                                                n_tokens - tpos,
5913                                                instr,
5914                                                data);
5915
5916         if (!strcmp(tokens[tpos], "jmplt"))
5917                 return instr_jmp_lt_translate(p,
5918                                               action,
5919                                               &tokens[tpos],
5920                                               n_tokens - tpos,
5921                                               instr,
5922                                               data);
5923
5924         if (!strcmp(tokens[tpos], "jmpgt"))
5925                 return instr_jmp_gt_translate(p,
5926                                               action,
5927                                               &tokens[tpos],
5928                                               n_tokens - tpos,
5929                                               instr,
5930                                               data);
5931
5932         if (!strcmp(tokens[tpos], "return"))
5933                 return instr_return_translate(p,
5934                                               action,
5935                                               &tokens[tpos],
5936                                               n_tokens - tpos,
5937                                               instr,
5938                                               data);
5939
5940         CHECK(0, EINVAL);
5941 }
5942
5943 static struct instruction_data *
5944 label_find(struct instruction_data *data, uint32_t n, const char *label)
5945 {
5946         uint32_t i;
5947
5948         for (i = 0; i < n; i++)
5949                 if (!strcmp(label, data[i].label))
5950                         return &data[i];
5951
5952         return NULL;
5953 }
5954
5955 static uint32_t
5956 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5957 {
5958         uint32_t count = 0, i;
5959
5960         if (!label[0])
5961                 return 0;
5962
5963         for (i = 0; i < n; i++)
5964                 if (!strcmp(label, data[i].jmp_label))
5965                         count++;
5966
5967         return count;
5968 }
5969
5970 static int
5971 instr_label_check(struct instruction_data *instruction_data,
5972                   uint32_t n_instructions)
5973 {
5974         uint32_t i;
5975
5976         /* Check that all instruction labels are unique. */
5977         for (i = 0; i < n_instructions; i++) {
5978                 struct instruction_data *data = &instruction_data[i];
5979                 char *label = data->label;
5980                 uint32_t j;
5981
5982                 if (!label[0])
5983                         continue;
5984
5985                 for (j = i + 1; j < n_instructions; j++)
5986                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
5987         }
5988
5989         /* Get users for each instruction label. */
5990         for (i = 0; i < n_instructions; i++) {
5991                 struct instruction_data *data = &instruction_data[i];
5992                 char *label = data->label;
5993
5994                 data->n_users = label_is_used(instruction_data,
5995                                               n_instructions,
5996                                               label);
5997         }
5998
5999         return 0;
6000 }
6001
6002 static int
6003 instr_jmp_resolve(struct instruction *instructions,
6004                   struct instruction_data *instruction_data,
6005                   uint32_t n_instructions)
6006 {
6007         uint32_t i;
6008
6009         for (i = 0; i < n_instructions; i++) {
6010                 struct instruction *instr = &instructions[i];
6011                 struct instruction_data *data = &instruction_data[i];
6012                 struct instruction_data *found;
6013
6014                 if (!instruction_is_jmp(instr))
6015                         continue;
6016
6017                 found = label_find(instruction_data,
6018                                    n_instructions,
6019                                    data->jmp_label);
6020                 CHECK(found, EINVAL);
6021
6022                 instr->jmp.ip = &instructions[found - instruction_data];
6023         }
6024
6025         return 0;
6026 }
6027
6028 static int
6029 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6030              struct action *a,
6031              struct instruction *instr,
6032              struct instruction_data *data __rte_unused,
6033              uint32_t n_instructions)
6034 {
6035         if (!a) {
6036                 enum instruction_type type;
6037                 uint32_t i;
6038
6039                 /* Check that the first instruction is rx. */
6040                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6041
6042                 /* Check that there is at least one tx instruction. */
6043                 for (i = 0; i < n_instructions; i++) {
6044                         type = instr[i].type;
6045
6046                         if (instruction_is_tx(type))
6047                                 break;
6048                 }
6049                 CHECK(i < n_instructions, EINVAL);
6050
6051                 /* Check that the last instruction is either tx or unconditional
6052                  * jump.
6053                  */
6054                 type = instr[n_instructions - 1].type;
6055                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6056         }
6057
6058         if (a) {
6059                 enum instruction_type type;
6060                 uint32_t i;
6061
6062                 /* Check that there is at least one return or tx instruction. */
6063                 for (i = 0; i < n_instructions; i++) {
6064                         type = instr[i].type;
6065
6066                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6067                                 break;
6068                 }
6069                 CHECK(i < n_instructions, EINVAL);
6070         }
6071
6072         return 0;
6073 }
6074
6075 static uint32_t
6076 instr_compact(struct instruction *instructions,
6077               struct instruction_data *instruction_data,
6078               uint32_t n_instructions)
6079 {
6080         uint32_t i, pos = 0;
6081
6082         /* Eliminate the invalid instructions that have been optimized out. */
6083         for (i = 0; i < n_instructions; i++) {
6084                 struct instruction *instr = &instructions[i];
6085                 struct instruction_data *data = &instruction_data[i];
6086
6087                 if (data->invalid)
6088                         continue;
6089
6090                 if (i != pos) {
6091                         memcpy(&instructions[pos], instr, sizeof(*instr));
6092                         memcpy(&instruction_data[pos], data, sizeof(*data));
6093                 }
6094
6095                 pos++;
6096         }
6097
6098         return pos;
6099 }
6100
6101 static int
6102 instr_pattern_extract_many_search(struct instruction *instr,
6103                                   struct instruction_data *data,
6104                                   uint32_t n_instr,
6105                                   uint32_t *n_pattern_instr)
6106 {
6107         uint32_t i;
6108
6109         for (i = 0; i < n_instr; i++) {
6110                 if (data[i].invalid)
6111                         break;
6112
6113                 if (instr[i].type != INSTR_HDR_EXTRACT)
6114                         break;
6115
6116                 if (i == RTE_DIM(instr->io.hdr.header_id))
6117                         break;
6118
6119                 if (i && data[i].n_users)
6120                         break;
6121         }
6122
6123         if (i < 2)
6124                 return 0;
6125
6126         *n_pattern_instr = i;
6127         return 1;
6128 }
6129
6130 static void
6131 instr_pattern_extract_many_replace(struct instruction *instr,
6132                                    struct instruction_data *data,
6133                                    uint32_t n_instr)
6134 {
6135         uint32_t i;
6136
6137         for (i = 1; i < n_instr; i++) {
6138                 instr[0].type++;
6139                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6140                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6141                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6142
6143                 data[i].invalid = 1;
6144         }
6145 }
6146
6147 static uint32_t
6148 instr_pattern_extract_many_optimize(struct instruction *instructions,
6149                                     struct instruction_data *instruction_data,
6150                                     uint32_t n_instructions)
6151 {
6152         uint32_t i;
6153
6154         for (i = 0; i < n_instructions; ) {
6155                 struct instruction *instr = &instructions[i];
6156                 struct instruction_data *data = &instruction_data[i];
6157                 uint32_t n_instr = 0;
6158                 int detected;
6159
6160                 /* Extract many. */
6161                 detected = instr_pattern_extract_many_search(instr,
6162                                                              data,
6163                                                              n_instructions - i,
6164                                                              &n_instr);
6165                 if (detected) {
6166                         instr_pattern_extract_many_replace(instr,
6167                                                            data,
6168                                                            n_instr);
6169                         i += n_instr;
6170                         continue;
6171                 }
6172
6173                 /* No pattern starting at the current instruction. */
6174                 i++;
6175         }
6176
6177         /* Eliminate the invalid instructions that have been optimized out. */
6178         n_instructions = instr_compact(instructions,
6179                                        instruction_data,
6180                                        n_instructions);
6181
6182         return n_instructions;
6183 }
6184
6185 static int
6186 instr_pattern_emit_many_tx_search(struct instruction *instr,
6187                                   struct instruction_data *data,
6188                                   uint32_t n_instr,
6189                                   uint32_t *n_pattern_instr)
6190 {
6191         uint32_t i;
6192
6193         for (i = 0; i < n_instr; i++) {
6194                 if (data[i].invalid)
6195                         break;
6196
6197                 if (instr[i].type != INSTR_HDR_EMIT)
6198                         break;
6199
6200                 if (i == RTE_DIM(instr->io.hdr.header_id))
6201                         break;
6202
6203                 if (i && data[i].n_users)
6204                         break;
6205         }
6206
6207         if (!i)
6208                 return 0;
6209
6210         if (!instruction_is_tx(instr[i].type))
6211                 return 0;
6212
6213         if (data[i].n_users)
6214                 return 0;
6215
6216         i++;
6217
6218         *n_pattern_instr = i;
6219         return 1;
6220 }
6221
6222 static void
6223 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6224                                    struct instruction_data *data,
6225                                    uint32_t n_instr)
6226 {
6227         uint32_t i;
6228
6229         /* Any emit instruction in addition to the first one. */
6230         for (i = 1; i < n_instr - 1; i++) {
6231                 instr[0].type++;
6232                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6233                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6234                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6235
6236                 data[i].invalid = 1;
6237         }
6238
6239         /* The TX instruction is the last one in the pattern. */
6240         instr[0].type++;
6241         instr[0].io.io.offset = instr[i].io.io.offset;
6242         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6243         data[i].invalid = 1;
6244 }
6245
6246 static uint32_t
6247 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6248                                     struct instruction_data *instruction_data,
6249                                     uint32_t n_instructions)
6250 {
6251         uint32_t i;
6252
6253         for (i = 0; i < n_instructions; ) {
6254                 struct instruction *instr = &instructions[i];
6255                 struct instruction_data *data = &instruction_data[i];
6256                 uint32_t n_instr = 0;
6257                 int detected;
6258
6259                 /* Emit many + TX. */
6260                 detected = instr_pattern_emit_many_tx_search(instr,
6261                                                              data,
6262                                                              n_instructions - i,
6263                                                              &n_instr);
6264                 if (detected) {
6265                         instr_pattern_emit_many_tx_replace(instr,
6266                                                            data,
6267                                                            n_instr);
6268                         i += n_instr;
6269                         continue;
6270                 }
6271
6272                 /* No pattern starting at the current instruction. */
6273                 i++;
6274         }
6275
6276         /* Eliminate the invalid instructions that have been optimized out. */
6277         n_instructions = instr_compact(instructions,
6278                                        instruction_data,
6279                                        n_instructions);
6280
6281         return n_instructions;
6282 }
6283
6284 static uint32_t
6285 action_arg_src_mov_count(struct action *a,
6286                          uint32_t arg_id,
6287                          struct instruction *instructions,
6288                          struct instruction_data *instruction_data,
6289                          uint32_t n_instructions);
6290
6291 static int
6292 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6293                                       struct action *a,
6294                                       struct instruction *instr,
6295                                       struct instruction_data *data,
6296                                       uint32_t n_instr,
6297                                       struct instruction *instructions,
6298                                       struct instruction_data *instruction_data,
6299                                       uint32_t n_instructions,
6300                                       uint32_t *n_pattern_instr)
6301 {
6302         struct header *h;
6303         uint32_t src_field_id, i, j;
6304
6305         /* Prerequisites. */
6306         if (!a || !a->st)
6307                 return 0;
6308
6309         /* First instruction: MOV_HM. */
6310         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6311                 return 0;
6312
6313         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6314         if (!h || h->st->var_size)
6315                 return 0;
6316
6317         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6318                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6319                         break;
6320
6321         if (src_field_id == a->st->n_fields)
6322                 return 0;
6323
6324         if (instr[0].mov.dst.offset ||
6325             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6326             instr[0].mov.src.struct_id ||
6327             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6328             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6329                 return 0;
6330
6331         if ((n_instr < h->st->n_fields + 1) ||
6332              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6333                 return 0;
6334
6335         /* Subsequent instructions: MOV_HM. */
6336         for (i = 1; i < h->st->n_fields; i++)
6337                 if (data[i].invalid ||
6338                     data[i].n_users ||
6339                     (instr[i].type != INSTR_MOV_HM) ||
6340                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6341                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6342                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6343                     instr[i].mov.src.struct_id ||
6344                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6345                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6346                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6347                         return 0;
6348
6349         /* Last instruction: HDR_VALIDATE. */
6350         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6351             (instr[i].valid.header_id != h->id))
6352                 return 0;
6353
6354         /* Check that none of the action args that are used as source for this
6355          * DMA transfer are not used as source in any other mov instruction.
6356          */
6357         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6358                 uint32_t n_users;
6359
6360                 n_users = action_arg_src_mov_count(a,
6361                                                    j,
6362                                                    instructions,
6363                                                    instruction_data,
6364                                                    n_instructions);
6365                 if (n_users > 1)
6366                         return 0;
6367         }
6368
6369         *n_pattern_instr = 1 + i;
6370         return 1;
6371 }
6372
6373 static void
6374 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6375                                        struct action *a,
6376                                        struct instruction *instr,
6377                                        struct instruction_data *data,
6378                                        uint32_t n_instr)
6379 {
6380         struct header *h;
6381         uint32_t src_field_id, src_offset, i;
6382
6383         /* Read from the instructions before they are modified. */
6384         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6385         if (!h)
6386                 return;
6387
6388         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6389                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6390                         break;
6391
6392         if (src_field_id == a->st->n_fields)
6393                 return;
6394
6395         src_offset = instr[0].mov.src.offset;
6396
6397         /* Modify the instructions. */
6398         instr[0].type = INSTR_DMA_HT;
6399         instr[0].dma.dst.header_id[0] = h->id;
6400         instr[0].dma.dst.struct_id[0] = h->struct_id;
6401         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6402         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6403
6404         for (i = 1; i < n_instr; i++)
6405                 data[i].invalid = 1;
6406
6407         /* Update the endianness of the action arguments to header endianness. */
6408         for (i = 0; i < h->st->n_fields; i++)
6409                 a->args_endianness[src_field_id + i] = 1;
6410 }
6411
6412 static uint32_t
6413 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6414                                         struct action *a,
6415                                         struct instruction *instructions,
6416                                         struct instruction_data *instruction_data,
6417                                         uint32_t n_instructions)
6418 {
6419         uint32_t i;
6420
6421         if (!a || !a->st)
6422                 return n_instructions;
6423
6424         for (i = 0; i < n_instructions; ) {
6425                 struct instruction *instr = &instructions[i];
6426                 struct instruction_data *data = &instruction_data[i];
6427                 uint32_t n_instr = 0;
6428                 int detected;
6429
6430                 /* Mov all + validate. */
6431                 detected = instr_pattern_mov_all_validate_search(p,
6432                                                                  a,
6433                                                                  instr,
6434                                                                  data,
6435                                                                  n_instructions - i,
6436                                                                  instructions,
6437                                                                  instruction_data,
6438                                                                  n_instructions,
6439                                                                  &n_instr);
6440                 if (detected) {
6441                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6442                         i += n_instr;
6443                         continue;
6444                 }
6445
6446                 /* No pattern starting at the current instruction. */
6447                 i++;
6448         }
6449
6450         /* Eliminate the invalid instructions that have been optimized out. */
6451         n_instructions = instr_compact(instructions,
6452                                        instruction_data,
6453                                        n_instructions);
6454
6455         return n_instructions;
6456 }
6457
6458 static int
6459 instr_pattern_dma_many_search(struct instruction *instr,
6460                               struct instruction_data *data,
6461                               uint32_t n_instr,
6462                               uint32_t *n_pattern_instr)
6463 {
6464         uint32_t i;
6465
6466         for (i = 0; i < n_instr; i++) {
6467                 if (data[i].invalid)
6468                         break;
6469
6470                 if (instr[i].type != INSTR_DMA_HT)
6471                         break;
6472
6473                 if (i == RTE_DIM(instr->dma.dst.header_id))
6474                         break;
6475
6476                 if (i && data[i].n_users)
6477                         break;
6478         }
6479
6480         if (i < 2)
6481                 return 0;
6482
6483         *n_pattern_instr = i;
6484         return 1;
6485 }
6486
6487 static void
6488 instr_pattern_dma_many_replace(struct instruction *instr,
6489                                struct instruction_data *data,
6490                                uint32_t n_instr)
6491 {
6492         uint32_t i;
6493
6494         for (i = 1; i < n_instr; i++) {
6495                 instr[0].type++;
6496                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6497                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6498                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6499                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6500
6501                 data[i].invalid = 1;
6502         }
6503 }
6504
6505 static uint32_t
6506 instr_pattern_dma_many_optimize(struct instruction *instructions,
6507                struct instruction_data *instruction_data,
6508                uint32_t n_instructions)
6509 {
6510         uint32_t i;
6511
6512         for (i = 0; i < n_instructions; ) {
6513                 struct instruction *instr = &instructions[i];
6514                 struct instruction_data *data = &instruction_data[i];
6515                 uint32_t n_instr = 0;
6516                 int detected;
6517
6518                 /* DMA many. */
6519                 detected = instr_pattern_dma_many_search(instr,
6520                                                          data,
6521                                                          n_instructions - i,
6522                                                          &n_instr);
6523                 if (detected) {
6524                         instr_pattern_dma_many_replace(instr, data, n_instr);
6525                         i += n_instr;
6526                         continue;
6527                 }
6528
6529                 /* No pattern starting at the current instruction. */
6530                 i++;
6531         }
6532
6533         /* Eliminate the invalid instructions that have been optimized out. */
6534         n_instructions = instr_compact(instructions,
6535                                        instruction_data,
6536                                        n_instructions);
6537
6538         return n_instructions;
6539 }
6540
6541 static uint32_t
6542 instr_optimize(struct rte_swx_pipeline *p,
6543                struct action *a,
6544                struct instruction *instructions,
6545                struct instruction_data *instruction_data,
6546                uint32_t n_instructions)
6547 {
6548         /* Extract many. */
6549         n_instructions = instr_pattern_extract_many_optimize(instructions,
6550                                                              instruction_data,
6551                                                              n_instructions);
6552
6553         /* Emit many + TX. */
6554         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6555                                                              instruction_data,
6556                                                              n_instructions);
6557
6558         /* Mov all + validate. */
6559         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6560                                                                  a,
6561                                                                  instructions,
6562                                                                  instruction_data,
6563                                                                  n_instructions);
6564
6565         /* DMA many. */
6566         n_instructions = instr_pattern_dma_many_optimize(instructions,
6567                                                          instruction_data,
6568                                                          n_instructions);
6569
6570         return n_instructions;
6571 }
6572
6573 static int
6574 instruction_config(struct rte_swx_pipeline *p,
6575                    struct action *a,
6576                    const char **instructions,
6577                    uint32_t n_instructions)
6578 {
6579         struct instruction *instr = NULL;
6580         struct instruction_data *data = NULL;
6581         int err = 0;
6582         uint32_t i;
6583
6584         CHECK(n_instructions, EINVAL);
6585         CHECK(instructions, EINVAL);
6586         for (i = 0; i < n_instructions; i++)
6587                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6588
6589         /* Memory allocation. */
6590         instr = calloc(n_instructions, sizeof(struct instruction));
6591         if (!instr) {
6592                 err = -ENOMEM;
6593                 goto error;
6594         }
6595
6596         data = calloc(n_instructions, sizeof(struct instruction_data));
6597         if (!data) {
6598                 err = -ENOMEM;
6599                 goto error;
6600         }
6601
6602         for (i = 0; i < n_instructions; i++) {
6603                 char *string = strdup(instructions[i]);
6604                 if (!string) {
6605                         err = -ENOMEM;
6606                         goto error;
6607                 }
6608
6609                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6610                 if (err) {
6611                         free(string);
6612                         goto error;
6613                 }
6614
6615                 free(string);
6616         }
6617
6618         err = instr_label_check(data, n_instructions);
6619         if (err)
6620                 goto error;
6621
6622         err = instr_verify(p, a, instr, data, n_instructions);
6623         if (err)
6624                 goto error;
6625
6626         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6627
6628         err = instr_jmp_resolve(instr, data, n_instructions);
6629         if (err)
6630                 goto error;
6631
6632         if (a) {
6633                 a->instructions = instr;
6634                 a->instruction_data = data;
6635                 a->n_instructions = n_instructions;
6636         } else {
6637                 p->instructions = instr;
6638                 p->instruction_data = data;
6639                 p->n_instructions = n_instructions;
6640         }
6641
6642         return 0;
6643
6644 error:
6645         free(data);
6646         free(instr);
6647         return err;
6648 }
6649
6650 static instr_exec_t instruction_table[] = {
6651         [INSTR_RX] = instr_rx_exec,
6652         [INSTR_TX] = instr_tx_exec,
6653         [INSTR_TX_I] = instr_tx_i_exec,
6654
6655         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6656         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6657         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6658         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6659         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6660         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6661         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6662         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6663         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6664         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6665
6666         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6667         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6668         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6669         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6670         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6671         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6672         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6673         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6674         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6675
6676         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6677         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6678
6679         [INSTR_MOV] = instr_mov_exec,
6680         [INSTR_MOV_MH] = instr_mov_mh_exec,
6681         [INSTR_MOV_HM] = instr_mov_hm_exec,
6682         [INSTR_MOV_HH] = instr_mov_hh_exec,
6683         [INSTR_MOV_I] = instr_mov_i_exec,
6684
6685         [INSTR_DMA_HT] = instr_dma_ht_exec,
6686         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6687         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6688         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6689         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6690         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6691         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6692         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6693
6694         [INSTR_ALU_ADD] = instr_alu_add_exec,
6695         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6696         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6697         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6698         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6699         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6700
6701         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6702         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6703         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6704         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6705         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6706         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6707
6708         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6709         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6710         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6711         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6712
6713         [INSTR_ALU_AND] = instr_alu_and_exec,
6714         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6715         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6716         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6717         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6718
6719         [INSTR_ALU_OR] = instr_alu_or_exec,
6720         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6721         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6722         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6723         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6724
6725         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6726         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6727         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6728         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6729         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6730
6731         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6732         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6733         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6734         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6735         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6736         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6737
6738         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6739         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6740         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6741         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6742         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6743         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6744
6745         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6746         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6747         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6748
6749         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6750         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6751         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6752         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6753         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6754         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6755
6756         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6757         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6758         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6759         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6760         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6761         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6762         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6763         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6764         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6765
6766         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6767         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6768         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6769         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6770         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6771         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6772         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6773         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6774         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6775
6776         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6777         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6778         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6779
6780         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6781         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6782         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6783         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6784         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6785         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6786         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6787         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6788         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6789         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6790         [INSTR_METER_IMM] = instr_meter_imm_exec,
6791         [INSTR_METER_IMI] = instr_meter_imi_exec,
6792
6793         [INSTR_TABLE] = instr_table_exec,
6794         [INSTR_TABLE_AF] = instr_table_af_exec,
6795         [INSTR_SELECTOR] = instr_selector_exec,
6796         [INSTR_LEARNER] = instr_learner_exec,
6797         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6798         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6799         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6800         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6801         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6802
6803         [INSTR_JMP] = instr_jmp_exec,
6804         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6805         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6806         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6807         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6808         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6809         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6810
6811         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6812         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6813         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6814         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6815         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6816
6817         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6818         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6819         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6820         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6821         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6822
6823         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6824         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6825         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6826         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6827         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6828         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6829
6830         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6831         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6832         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6833         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6834         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6835         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6836
6837         [INSTR_RETURN] = instr_return_exec,
6838 };
6839
6840 static int
6841 instruction_table_build(struct rte_swx_pipeline *p)
6842 {
6843         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6844                                       sizeof(struct instr_exec_t *));
6845         if (!p->instruction_table)
6846                 return -EINVAL;
6847
6848         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6849
6850         return 0;
6851 }
6852
6853 static void
6854 instruction_table_build_free(struct rte_swx_pipeline *p)
6855 {
6856         if (!p->instruction_table)
6857                 return;
6858
6859         free(p->instruction_table);
6860         p->instruction_table = NULL;
6861 }
6862
6863 static void
6864 instruction_table_free(struct rte_swx_pipeline *p)
6865 {
6866         instruction_table_build_free(p);
6867 }
6868
6869 static inline void
6870 instr_exec(struct rte_swx_pipeline *p)
6871 {
6872         struct thread *t = &p->threads[p->thread_id];
6873         struct instruction *ip = t->ip;
6874         instr_exec_t instr = p->instruction_table[ip->type];
6875
6876         instr(p);
6877 }
6878
6879 /*
6880  * Action.
6881  */
6882 static struct action *
6883 action_find(struct rte_swx_pipeline *p, const char *name)
6884 {
6885         struct action *elem;
6886
6887         if (!name)
6888                 return NULL;
6889
6890         TAILQ_FOREACH(elem, &p->actions, node)
6891                 if (strcmp(elem->name, name) == 0)
6892                         return elem;
6893
6894         return NULL;
6895 }
6896
6897 static struct action *
6898 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6899 {
6900         struct action *action = NULL;
6901
6902         TAILQ_FOREACH(action, &p->actions, node)
6903                 if (action->id == id)
6904                         return action;
6905
6906         return NULL;
6907 }
6908
6909 static struct field *
6910 action_field_find(struct action *a, const char *name)
6911 {
6912         return a->st ? struct_type_field_find(a->st, name) : NULL;
6913 }
6914
6915 static struct field *
6916 action_field_parse(struct action *action, const char *name)
6917 {
6918         if (name[0] != 't' || name[1] != '.')
6919                 return NULL;
6920
6921         return action_field_find(action, &name[2]);
6922 }
6923
6924 static int
6925 action_has_nbo_args(struct action *a)
6926 {
6927         uint32_t i;
6928
6929         /* Return if the action does not have any args. */
6930         if (!a->st)
6931                 return 0; /* FALSE */
6932
6933         for (i = 0; i < a->st->n_fields; i++)
6934                 if (a->args_endianness[i])
6935                         return 1; /* TRUE */
6936
6937         return 0; /* FALSE */
6938 }
6939
6940 static int
6941 action_does_learning(struct action *a)
6942 {
6943         uint32_t i;
6944
6945         for (i = 0; i < a->n_instructions; i++)
6946                 switch (a->instructions[i].type) {
6947                 case INSTR_LEARNER_LEARN:
6948                         return 1; /* TRUE */
6949
6950                 case INSTR_LEARNER_FORGET:
6951                         return 1; /* TRUE */
6952
6953                 default:
6954                         continue;
6955                 }
6956
6957         return 0; /* FALSE */
6958 }
6959
6960 int
6961 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6962                                const char *name,
6963                                const char *args_struct_type_name,
6964                                const char **instructions,
6965                                uint32_t n_instructions)
6966 {
6967         struct struct_type *args_struct_type = NULL;
6968         struct action *a;
6969         int err;
6970
6971         CHECK(p, EINVAL);
6972
6973         CHECK_NAME(name, EINVAL);
6974         CHECK(!action_find(p, name), EEXIST);
6975
6976         if (args_struct_type_name) {
6977                 CHECK_NAME(args_struct_type_name, EINVAL);
6978                 args_struct_type = struct_type_find(p, args_struct_type_name);
6979                 CHECK(args_struct_type, EINVAL);
6980                 CHECK(!args_struct_type->var_size, EINVAL);
6981         }
6982
6983         /* Node allocation. */
6984         a = calloc(1, sizeof(struct action));
6985         CHECK(a, ENOMEM);
6986         if (args_struct_type) {
6987                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6988                 if (!a->args_endianness) {
6989                         free(a);
6990                         CHECK(0, ENOMEM);
6991                 }
6992         }
6993
6994         /* Node initialization. */
6995         strcpy(a->name, name);
6996         a->st = args_struct_type;
6997         a->id = p->n_actions;
6998
6999         /* Instruction translation. */
7000         err = instruction_config(p, a, instructions, n_instructions);
7001         if (err) {
7002                 free(a->args_endianness);
7003                 free(a);
7004                 return err;
7005         }
7006
7007         /* Node add to tailq. */
7008         TAILQ_INSERT_TAIL(&p->actions, a, node);
7009         p->n_actions++;
7010
7011         return 0;
7012 }
7013
7014 static int
7015 action_build(struct rte_swx_pipeline *p)
7016 {
7017         struct action *action;
7018
7019         /* p->action_instructions. */
7020         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7021         CHECK(p->action_instructions, ENOMEM);
7022
7023         TAILQ_FOREACH(action, &p->actions, node)
7024                 p->action_instructions[action->id] = action->instructions;
7025
7026         /* p->action_funcs. */
7027         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7028         CHECK(p->action_funcs, ENOMEM);
7029
7030         return 0;
7031 }
7032
7033 static void
7034 action_build_free(struct rte_swx_pipeline *p)
7035 {
7036         free(p->action_funcs);
7037         p->action_funcs = NULL;
7038
7039         free(p->action_instructions);
7040         p->action_instructions = NULL;
7041 }
7042
7043 static void
7044 action_free(struct rte_swx_pipeline *p)
7045 {
7046         action_build_free(p);
7047
7048         for ( ; ; ) {
7049                 struct action *action;
7050
7051                 action = TAILQ_FIRST(&p->actions);
7052                 if (!action)
7053                         break;
7054
7055                 TAILQ_REMOVE(&p->actions, action, node);
7056                 free(action->instruction_data);
7057                 free(action->instructions);
7058                 free(action);
7059         }
7060 }
7061
7062 static uint32_t
7063 action_arg_src_mov_count(struct action *a,
7064                          uint32_t arg_id,
7065                          struct instruction *instructions,
7066                          struct instruction_data *instruction_data,
7067                          uint32_t n_instructions)
7068 {
7069         uint32_t offset, n_users = 0, i;
7070
7071         if (!a->st ||
7072             (arg_id >= a->st->n_fields) ||
7073             !instructions ||
7074             !instruction_data ||
7075             !n_instructions)
7076                 return 0;
7077
7078         offset = a->st->fields[arg_id].offset / 8;
7079
7080         for (i = 0; i < n_instructions; i++) {
7081                 struct instruction *instr = &instructions[i];
7082                 struct instruction_data *data = &instruction_data[i];
7083
7084                 if (data->invalid ||
7085                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7086                     instr->mov.src.struct_id ||
7087                     (instr->mov.src.offset != offset))
7088                         continue;
7089
7090                 n_users++;
7091         }
7092
7093         return n_users;
7094 }
7095
7096 /*
7097  * Table.
7098  */
7099 static struct table_type *
7100 table_type_find(struct rte_swx_pipeline *p, const char *name)
7101 {
7102         struct table_type *elem;
7103
7104         TAILQ_FOREACH(elem, &p->table_types, node)
7105                 if (strcmp(elem->name, name) == 0)
7106                         return elem;
7107
7108         return NULL;
7109 }
7110
7111 static struct table_type *
7112 table_type_resolve(struct rte_swx_pipeline *p,
7113                    const char *recommended_type_name,
7114                    enum rte_swx_table_match_type match_type)
7115 {
7116         struct table_type *elem;
7117
7118         /* Only consider the recommended type if the match type is correct. */
7119         if (recommended_type_name)
7120                 TAILQ_FOREACH(elem, &p->table_types, node)
7121                         if (!strcmp(elem->name, recommended_type_name) &&
7122                             (elem->match_type == match_type))
7123                                 return elem;
7124
7125         /* Ignore the recommended type and get the first element with this match
7126          * type.
7127          */
7128         TAILQ_FOREACH(elem, &p->table_types, node)
7129                 if (elem->match_type == match_type)
7130                         return elem;
7131
7132         return NULL;
7133 }
7134
7135 static struct table *
7136 table_find(struct rte_swx_pipeline *p, const char *name)
7137 {
7138         struct table *elem;
7139
7140         TAILQ_FOREACH(elem, &p->tables, node)
7141                 if (strcmp(elem->name, name) == 0)
7142                         return elem;
7143
7144         return NULL;
7145 }
7146
7147 static struct table *
7148 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7149 {
7150         struct table *table = NULL;
7151
7152         TAILQ_FOREACH(table, &p->tables, node)
7153                 if (table->id == id)
7154                         return table;
7155
7156         return NULL;
7157 }
7158
7159 int
7160 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7161                                      const char *name,
7162                                      enum rte_swx_table_match_type match_type,
7163                                      struct rte_swx_table_ops *ops)
7164 {
7165         struct table_type *elem;
7166
7167         CHECK(p, EINVAL);
7168
7169         CHECK_NAME(name, EINVAL);
7170         CHECK(!table_type_find(p, name), EEXIST);
7171
7172         CHECK(ops, EINVAL);
7173         CHECK(ops->create, EINVAL);
7174         CHECK(ops->lkp, EINVAL);
7175         CHECK(ops->free, EINVAL);
7176
7177         /* Node allocation. */
7178         elem = calloc(1, sizeof(struct table_type));
7179         CHECK(elem, ENOMEM);
7180
7181         /* Node initialization. */
7182         strcpy(elem->name, name);
7183         elem->match_type = match_type;
7184         memcpy(&elem->ops, ops, sizeof(*ops));
7185
7186         /* Node add to tailq. */
7187         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7188
7189         return 0;
7190 }
7191
7192 static int
7193 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7194                          uint32_t n_fields,
7195                          enum rte_swx_table_match_type *match_type)
7196 {
7197         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7198
7199         for (i = 0; i < n_fields; i++) {
7200                 struct rte_swx_match_field_params  *f = &fields[i];
7201
7202                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7203                         n_fields_em++;
7204
7205                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7206                         n_fields_lpm++;
7207         }
7208
7209         if ((n_fields_lpm > 1) ||
7210             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7211                 return -EINVAL;
7212
7213         *match_type = (n_fields_em == n_fields) ?
7214                        RTE_SWX_TABLE_MATCH_EXACT :
7215                        RTE_SWX_TABLE_MATCH_WILDCARD;
7216
7217         return 0;
7218 }
7219
7220 static int
7221 table_match_fields_check(struct rte_swx_pipeline *p,
7222                          struct rte_swx_pipeline_table_params *params,
7223                          struct header **header)
7224 {
7225         struct header *h0 = NULL;
7226         struct field *hf, *mf;
7227         uint32_t *offset = NULL, i;
7228         int status = 0;
7229
7230         /* Return if no match fields. */
7231         if (!params->n_fields) {
7232                 if (params->fields) {
7233                         status = -EINVAL;
7234                         goto end;
7235                 }
7236
7237                 if (header)
7238                         *header = NULL;
7239
7240                 return 0;
7241         }
7242
7243         /* Memory allocation. */
7244         offset = calloc(params->n_fields, sizeof(uint32_t));
7245         if (!offset) {
7246                 status = -ENOMEM;
7247                 goto end;
7248         }
7249
7250         /* Check that all the match fields belong to either the same header or
7251          * to the meta-data.
7252          */
7253         hf = header_field_parse(p, params->fields[0].name, &h0);
7254         mf = metadata_field_parse(p, params->fields[0].name);
7255         if ((!hf && !mf) || (hf && hf->var_size)) {
7256                 status = -EINVAL;
7257                 goto end;
7258         }
7259
7260         offset[0] = h0 ? hf->offset : mf->offset;
7261
7262         for (i = 1; i < params->n_fields; i++)
7263                 if (h0) {
7264                         struct header *h;
7265
7266                         hf = header_field_parse(p, params->fields[i].name, &h);
7267                         if (!hf || (h->id != h0->id) || hf->var_size) {
7268                                 status = -EINVAL;
7269                                 goto end;
7270                         }
7271
7272                         offset[i] = hf->offset;
7273                 } else {
7274                         mf = metadata_field_parse(p, params->fields[i].name);
7275                         if (!mf) {
7276                                 status = -EINVAL;
7277                                 goto end;
7278                         }
7279
7280                         offset[i] = mf->offset;
7281                 }
7282
7283         /* Check that there are no duplicated match fields. */
7284         for (i = 0; i < params->n_fields; i++) {
7285                 uint32_t j;
7286
7287                 for (j = 0; j < i; j++)
7288                         if (offset[j] == offset[i]) {
7289                                 status = -EINVAL;
7290                                 goto end;
7291                         }
7292         }
7293
7294         /* Return. */
7295         if (header)
7296                 *header = h0;
7297
7298 end:
7299         free(offset);
7300         return status;
7301 }
7302
7303 int
7304 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7305                               const char *name,
7306                               struct rte_swx_pipeline_table_params *params,
7307                               const char *recommended_table_type_name,
7308                               const char *args,
7309                               uint32_t size)
7310 {
7311         struct table_type *type;
7312         struct table *t;
7313         struct action *default_action;
7314         struct header *header = NULL;
7315         uint32_t action_data_size_max = 0, i;
7316         int status = 0;
7317
7318         CHECK(p, EINVAL);
7319
7320         CHECK_NAME(name, EINVAL);
7321         CHECK(!table_find(p, name), EEXIST);
7322         CHECK(!selector_find(p, name), EEXIST);
7323         CHECK(!learner_find(p, name), EEXIST);
7324
7325         CHECK(params, EINVAL);
7326
7327         /* Match checks. */
7328         status = table_match_fields_check(p, params, &header);
7329         if (status)
7330                 return status;
7331
7332         /* Action checks. */
7333         CHECK(params->n_actions, EINVAL);
7334         CHECK(params->action_names, EINVAL);
7335         for (i = 0; i < params->n_actions; i++) {
7336                 const char *action_name = params->action_names[i];
7337                 struct action *a;
7338                 uint32_t action_data_size;
7339
7340                 CHECK_NAME(action_name, EINVAL);
7341
7342                 a = action_find(p, action_name);
7343                 CHECK(a, EINVAL);
7344                 CHECK(!action_does_learning(a), EINVAL);
7345
7346                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7347                 if (action_data_size > action_data_size_max)
7348                         action_data_size_max = action_data_size;
7349         }
7350
7351         CHECK_NAME(params->default_action_name, EINVAL);
7352         for (i = 0; i < p->n_actions; i++)
7353                 if (!strcmp(params->action_names[i],
7354                             params->default_action_name))
7355                         break;
7356         CHECK(i < params->n_actions, EINVAL);
7357         default_action = action_find(p, params->default_action_name);
7358         CHECK((default_action->st && params->default_action_data) ||
7359               !params->default_action_data, EINVAL);
7360
7361         /* Table type checks. */
7362         if (recommended_table_type_name)
7363                 CHECK_NAME(recommended_table_type_name, EINVAL);
7364
7365         if (params->n_fields) {
7366                 enum rte_swx_table_match_type match_type;
7367
7368                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7369                 if (status)
7370                         return status;
7371
7372                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7373                 CHECK(type, EINVAL);
7374         } else {
7375                 type = NULL;
7376         }
7377
7378         /* Memory allocation. */
7379         t = calloc(1, sizeof(struct table));
7380         CHECK(t, ENOMEM);
7381
7382         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7383         if (!t->fields) {
7384                 free(t);
7385                 CHECK(0, ENOMEM);
7386         }
7387
7388         t->actions = calloc(params->n_actions, sizeof(struct action *));
7389         if (!t->actions) {
7390                 free(t->fields);
7391                 free(t);
7392                 CHECK(0, ENOMEM);
7393         }
7394
7395         if (action_data_size_max) {
7396                 t->default_action_data = calloc(1, action_data_size_max);
7397                 if (!t->default_action_data) {
7398                         free(t->actions);
7399                         free(t->fields);
7400                         free(t);
7401                         CHECK(0, ENOMEM);
7402                 }
7403         }
7404
7405         /* Node initialization. */
7406         strcpy(t->name, name);
7407         if (args && args[0])
7408                 strcpy(t->args, args);
7409         t->type = type;
7410
7411         for (i = 0; i < params->n_fields; i++) {
7412                 struct rte_swx_match_field_params *field = &params->fields[i];
7413                 struct match_field *f = &t->fields[i];
7414
7415                 f->match_type = field->match_type;
7416                 f->field = header ?
7417                         header_field_parse(p, field->name, NULL) :
7418                         metadata_field_parse(p, field->name);
7419         }
7420         t->n_fields = params->n_fields;
7421         t->header = header;
7422
7423         for (i = 0; i < params->n_actions; i++)
7424                 t->actions[i] = action_find(p, params->action_names[i]);
7425         t->default_action = default_action;
7426         if (default_action->st)
7427                 memcpy(t->default_action_data,
7428                        params->default_action_data,
7429                        default_action->st->n_bits / 8);
7430         t->n_actions = params->n_actions;
7431         t->default_action_is_const = params->default_action_is_const;
7432         t->action_data_size_max = action_data_size_max;
7433
7434         t->size = size;
7435         t->id = p->n_tables;
7436
7437         /* Node add to tailq. */
7438         TAILQ_INSERT_TAIL(&p->tables, t, node);
7439         p->n_tables++;
7440
7441         return 0;
7442 }
7443
7444 static struct rte_swx_table_params *
7445 table_params_get(struct table *table)
7446 {
7447         struct rte_swx_table_params *params;
7448         struct field *first, *last;
7449         uint8_t *key_mask;
7450         uint32_t key_size, key_offset, action_data_size, i;
7451
7452         /* Memory allocation. */
7453         params = calloc(1, sizeof(struct rte_swx_table_params));
7454         if (!params)
7455                 return NULL;
7456
7457         /* Find first (smallest offset) and last (biggest offset) match fields. */
7458         first = table->fields[0].field;
7459         last = table->fields[0].field;
7460
7461         for (i = 0; i < table->n_fields; i++) {
7462                 struct field *f = table->fields[i].field;
7463
7464                 if (f->offset < first->offset)
7465                         first = f;
7466
7467                 if (f->offset > last->offset)
7468                         last = f;
7469         }
7470
7471         /* Key offset and size. */
7472         key_offset = first->offset / 8;
7473         key_size = (last->offset + last->n_bits - first->offset) / 8;
7474
7475         /* Memory allocation. */
7476         key_mask = calloc(1, key_size);
7477         if (!key_mask) {
7478                 free(params);
7479                 return NULL;
7480         }
7481
7482         /* Key mask. */
7483         for (i = 0; i < table->n_fields; i++) {
7484                 struct field *f = table->fields[i].field;
7485                 uint32_t start = (f->offset - first->offset) / 8;
7486                 size_t size = f->n_bits / 8;
7487
7488                 memset(&key_mask[start], 0xFF, size);
7489         }
7490
7491         /* Action data size. */
7492         action_data_size = 0;
7493         for (i = 0; i < table->n_actions; i++) {
7494                 struct action *action = table->actions[i];
7495                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7496
7497                 if (ads > action_data_size)
7498                         action_data_size = ads;
7499         }
7500
7501         /* Fill in. */
7502         params->match_type = table->type->match_type;
7503         params->key_size = key_size;
7504         params->key_offset = key_offset;
7505         params->key_mask0 = key_mask;
7506         params->action_data_size = action_data_size;
7507         params->n_keys_max = table->size;
7508
7509         return params;
7510 }
7511
7512 static void
7513 table_params_free(struct rte_swx_table_params *params)
7514 {
7515         if (!params)
7516                 return;
7517
7518         free(params->key_mask0);
7519         free(params);
7520 }
7521
7522 static int
7523 table_stub_lkp(void *table __rte_unused,
7524                void *mailbox __rte_unused,
7525                uint8_t **key __rte_unused,
7526                uint64_t *action_id __rte_unused,
7527                uint8_t **action_data __rte_unused,
7528                int *hit)
7529 {
7530         *hit = 0;
7531         return 1; /* DONE. */
7532 }
7533
7534 static int
7535 table_build(struct rte_swx_pipeline *p)
7536 {
7537         uint32_t i;
7538
7539         /* Per pipeline: table statistics. */
7540         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7541         CHECK(p->table_stats, ENOMEM);
7542
7543         for (i = 0; i < p->n_tables; i++) {
7544                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7545                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7546         }
7547
7548         /* Per thread: table runt-time. */
7549         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7550                 struct thread *t = &p->threads[i];
7551                 struct table *table;
7552
7553                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7554                 CHECK(t->tables, ENOMEM);
7555
7556                 TAILQ_FOREACH(table, &p->tables, node) {
7557                         struct table_runtime *r = &t->tables[table->id];
7558
7559                         if (table->type) {
7560                                 uint64_t size;
7561
7562                                 size = table->type->ops.mailbox_size_get();
7563
7564                                 /* r->func. */
7565                                 r->func = table->type->ops.lkp;
7566
7567                                 /* r->mailbox. */
7568                                 if (size) {
7569                                         r->mailbox = calloc(1, size);
7570                                         CHECK(r->mailbox, ENOMEM);
7571                                 }
7572
7573                                 /* r->key. */
7574                                 r->key = table->header ?
7575                                         &t->structs[table->header->struct_id] :
7576                                         &t->structs[p->metadata_struct_id];
7577                         } else {
7578                                 r->func = table_stub_lkp;
7579                         }
7580                 }
7581         }
7582
7583         return 0;
7584 }
7585
7586 static void
7587 table_build_free(struct rte_swx_pipeline *p)
7588 {
7589         uint32_t i;
7590
7591         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7592                 struct thread *t = &p->threads[i];
7593                 uint32_t j;
7594
7595                 if (!t->tables)
7596                         continue;
7597
7598                 for (j = 0; j < p->n_tables; j++) {
7599                         struct table_runtime *r = &t->tables[j];
7600
7601                         free(r->mailbox);
7602                 }
7603
7604                 free(t->tables);
7605                 t->tables = NULL;
7606         }
7607
7608         if (p->table_stats) {
7609                 for (i = 0; i < p->n_tables; i++)
7610                         free(p->table_stats[i].n_pkts_action);
7611
7612                 free(p->table_stats);
7613         }
7614 }
7615
7616 static void
7617 table_free(struct rte_swx_pipeline *p)
7618 {
7619         table_build_free(p);
7620
7621         /* Tables. */
7622         for ( ; ; ) {
7623                 struct table *elem;
7624
7625                 elem = TAILQ_FIRST(&p->tables);
7626                 if (!elem)
7627                         break;
7628
7629                 TAILQ_REMOVE(&p->tables, elem, node);
7630                 free(elem->fields);
7631                 free(elem->actions);
7632                 free(elem->default_action_data);
7633                 free(elem);
7634         }
7635
7636         /* Table types. */
7637         for ( ; ; ) {
7638                 struct table_type *elem;
7639
7640                 elem = TAILQ_FIRST(&p->table_types);
7641                 if (!elem)
7642                         break;
7643
7644                 TAILQ_REMOVE(&p->table_types, elem, node);
7645                 free(elem);
7646         }
7647 }
7648
7649 /*
7650  * Selector.
7651  */
7652 static struct selector *
7653 selector_find(struct rte_swx_pipeline *p, const char *name)
7654 {
7655         struct selector *s;
7656
7657         TAILQ_FOREACH(s, &p->selectors, node)
7658                 if (strcmp(s->name, name) == 0)
7659                         return s;
7660
7661         return NULL;
7662 }
7663
7664 static struct selector *
7665 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7666 {
7667         struct selector *s = NULL;
7668
7669         TAILQ_FOREACH(s, &p->selectors, node)
7670                 if (s->id == id)
7671                         return s;
7672
7673         return NULL;
7674 }
7675
7676 static int
7677 selector_fields_check(struct rte_swx_pipeline *p,
7678                       struct rte_swx_pipeline_selector_params *params,
7679                       struct header **header)
7680 {
7681         struct header *h0 = NULL;
7682         struct field *hf, *mf;
7683         uint32_t i;
7684
7685         /* Return if no selector fields. */
7686         if (!params->n_selector_fields || !params->selector_field_names)
7687                 return -EINVAL;
7688
7689         /* Check that all the selector fields either belong to the same header
7690          * or are all meta-data fields.
7691          */
7692         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7693         mf = metadata_field_parse(p, params->selector_field_names[0]);
7694         if (!hf && !mf)
7695                 return -EINVAL;
7696
7697         for (i = 1; i < params->n_selector_fields; i++)
7698                 if (h0) {
7699                         struct header *h;
7700
7701                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7702                         if (!hf || (h->id != h0->id))
7703                                 return -EINVAL;
7704                 } else {
7705                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7706                         if (!mf)
7707                                 return -EINVAL;
7708                 }
7709
7710         /* Check that there are no duplicated match fields. */
7711         for (i = 0; i < params->n_selector_fields; i++) {
7712                 const char *field_name = params->selector_field_names[i];
7713                 uint32_t j;
7714
7715                 for (j = i + 1; j < params->n_selector_fields; j++)
7716                         if (!strcmp(params->selector_field_names[j], field_name))
7717                                 return -EINVAL;
7718         }
7719
7720         /* Return. */
7721         if (header)
7722                 *header = h0;
7723
7724         return 0;
7725 }
7726
7727 int
7728 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7729                                  const char *name,
7730                                  struct rte_swx_pipeline_selector_params *params)
7731 {
7732         struct selector *s;
7733         struct header *selector_header = NULL;
7734         struct field *group_id_field, *member_id_field;
7735         uint32_t i;
7736         int status = 0;
7737
7738         CHECK(p, EINVAL);
7739
7740         CHECK_NAME(name, EINVAL);
7741         CHECK(!table_find(p, name), EEXIST);
7742         CHECK(!selector_find(p, name), EEXIST);
7743         CHECK(!learner_find(p, name), EEXIST);
7744
7745         CHECK(params, EINVAL);
7746
7747         CHECK_NAME(params->group_id_field_name, EINVAL);
7748         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7749         CHECK(group_id_field, EINVAL);
7750
7751         for (i = 0; i < params->n_selector_fields; i++) {
7752                 const char *field_name = params->selector_field_names[i];
7753
7754                 CHECK_NAME(field_name, EINVAL);
7755         }
7756         status = selector_fields_check(p, params, &selector_header);
7757         if (status)
7758                 return status;
7759
7760         CHECK_NAME(params->member_id_field_name, EINVAL);
7761         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7762         CHECK(member_id_field, EINVAL);
7763
7764         CHECK(params->n_groups_max, EINVAL);
7765
7766         CHECK(params->n_members_per_group_max, EINVAL);
7767
7768         /* Memory allocation. */
7769         s = calloc(1, sizeof(struct selector));
7770         if (!s) {
7771                 status = -ENOMEM;
7772                 goto error;
7773         }
7774
7775         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7776         if (!s->selector_fields) {
7777                 status = -ENOMEM;
7778                 goto error;
7779         }
7780
7781         /* Node initialization. */
7782         strcpy(s->name, name);
7783
7784         s->group_id_field = group_id_field;
7785
7786         for (i = 0; i < params->n_selector_fields; i++) {
7787                 const char *field_name = params->selector_field_names[i];
7788
7789                 s->selector_fields[i] = selector_header ?
7790                         header_field_parse(p, field_name, NULL) :
7791                         metadata_field_parse(p, field_name);
7792         }
7793
7794         s->n_selector_fields = params->n_selector_fields;
7795
7796         s->selector_header = selector_header;
7797
7798         s->member_id_field = member_id_field;
7799
7800         s->n_groups_max = params->n_groups_max;
7801
7802         s->n_members_per_group_max = params->n_members_per_group_max;
7803
7804         s->id = p->n_selectors;
7805
7806         /* Node add to tailq. */
7807         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7808         p->n_selectors++;
7809
7810         return 0;
7811
7812 error:
7813         if (!s)
7814                 return status;
7815
7816         free(s->selector_fields);
7817
7818         free(s);
7819
7820         return status;
7821 }
7822
7823 static void
7824 selector_params_free(struct rte_swx_table_selector_params *params)
7825 {
7826         if (!params)
7827                 return;
7828
7829         free(params->selector_mask);
7830
7831         free(params);
7832 }
7833
7834 static struct rte_swx_table_selector_params *
7835 selector_table_params_get(struct selector *s)
7836 {
7837         struct rte_swx_table_selector_params *params = NULL;
7838         struct field *first, *last;
7839         uint32_t i;
7840
7841         /* Memory allocation. */
7842         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7843         if (!params)
7844                 goto error;
7845
7846         /* Group ID. */
7847         params->group_id_offset = s->group_id_field->offset / 8;
7848
7849         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7850         first = s->selector_fields[0];
7851         last = s->selector_fields[0];
7852
7853         for (i = 0; i < s->n_selector_fields; i++) {
7854                 struct field *f = s->selector_fields[i];
7855
7856                 if (f->offset < first->offset)
7857                         first = f;
7858
7859                 if (f->offset > last->offset)
7860                         last = f;
7861         }
7862
7863         /* Selector offset and size. */
7864         params->selector_offset = first->offset / 8;
7865         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7866
7867         /* Memory allocation. */
7868         params->selector_mask = calloc(1, params->selector_size);
7869         if (!params->selector_mask)
7870                 goto error;
7871
7872         /* Selector mask. */
7873         for (i = 0; i < s->n_selector_fields; i++) {
7874                 struct field *f = s->selector_fields[i];
7875                 uint32_t start = (f->offset - first->offset) / 8;
7876                 size_t size = f->n_bits / 8;
7877
7878                 memset(&params->selector_mask[start], 0xFF, size);
7879         }
7880
7881         /* Member ID. */
7882         params->member_id_offset = s->member_id_field->offset / 8;
7883
7884         /* Maximum number of groups. */
7885         params->n_groups_max = s->n_groups_max;
7886
7887         /* Maximum number of members per group. */
7888         params->n_members_per_group_max = s->n_members_per_group_max;
7889
7890         return params;
7891
7892 error:
7893         selector_params_free(params);
7894         return NULL;
7895 }
7896
7897 static void
7898 selector_build_free(struct rte_swx_pipeline *p)
7899 {
7900         uint32_t i;
7901
7902         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7903                 struct thread *t = &p->threads[i];
7904                 uint32_t j;
7905
7906                 if (!t->selectors)
7907                         continue;
7908
7909                 for (j = 0; j < p->n_selectors; j++) {
7910                         struct selector_runtime *r = &t->selectors[j];
7911
7912                         free(r->mailbox);
7913                 }
7914
7915                 free(t->selectors);
7916                 t->selectors = NULL;
7917         }
7918
7919         free(p->selector_stats);
7920         p->selector_stats = NULL;
7921 }
7922
7923 static int
7924 selector_build(struct rte_swx_pipeline *p)
7925 {
7926         uint32_t i;
7927         int status = 0;
7928
7929         /* Per pipeline: selector statistics. */
7930         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7931         if (!p->selector_stats) {
7932                 status = -ENOMEM;
7933                 goto error;
7934         }
7935
7936         /* Per thread: selector run-time. */
7937         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7938                 struct thread *t = &p->threads[i];
7939                 struct selector *s;
7940
7941                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7942                 if (!t->selectors) {
7943                         status = -ENOMEM;
7944                         goto error;
7945                 }
7946
7947                 TAILQ_FOREACH(s, &p->selectors, node) {
7948                         struct selector_runtime *r = &t->selectors[s->id];
7949                         uint64_t size;
7950
7951                         /* r->mailbox. */
7952                         size = rte_swx_table_selector_mailbox_size_get();
7953                         if (size) {
7954                                 r->mailbox = calloc(1, size);
7955                                 if (!r->mailbox) {
7956                                         status = -ENOMEM;
7957                                         goto error;
7958                                 }
7959                         }
7960
7961                         /* r->group_id_buffer. */
7962                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
7963
7964                         /* r->selector_buffer. */
7965                         r->selector_buffer = s->selector_header ?
7966                                 &t->structs[s->selector_header->struct_id] :
7967                                 &t->structs[p->metadata_struct_id];
7968
7969                         /* r->member_id_buffer. */
7970                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
7971                 }
7972         }
7973
7974         return 0;
7975
7976 error:
7977         selector_build_free(p);
7978         return status;
7979 }
7980
7981 static void
7982 selector_free(struct rte_swx_pipeline *p)
7983 {
7984         selector_build_free(p);
7985
7986         /* Selector tables. */
7987         for ( ; ; ) {
7988                 struct selector *elem;
7989
7990                 elem = TAILQ_FIRST(&p->selectors);
7991                 if (!elem)
7992                         break;
7993
7994                 TAILQ_REMOVE(&p->selectors, elem, node);
7995                 free(elem->selector_fields);
7996                 free(elem);
7997         }
7998 }
7999
8000 /*
8001  * Learner table.
8002  */
8003 static struct learner *
8004 learner_find(struct rte_swx_pipeline *p, const char *name)
8005 {
8006         struct learner *l;
8007
8008         TAILQ_FOREACH(l, &p->learners, node)
8009                 if (!strcmp(l->name, name))
8010                         return l;
8011
8012         return NULL;
8013 }
8014
8015 static struct learner *
8016 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8017 {
8018         struct learner *l = NULL;
8019
8020         TAILQ_FOREACH(l, &p->learners, node)
8021                 if (l->id == id)
8022                         return l;
8023
8024         return NULL;
8025 }
8026
8027 static int
8028 learner_match_fields_check(struct rte_swx_pipeline *p,
8029                            struct rte_swx_pipeline_learner_params *params,
8030                            struct header **header)
8031 {
8032         struct header *h0 = NULL;
8033         struct field *hf, *mf;
8034         uint32_t i;
8035
8036         /* Return if no match fields. */
8037         if (!params->n_fields || !params->field_names)
8038                 return -EINVAL;
8039
8040         /* Check that all the match fields either belong to the same header
8041          * or are all meta-data fields.
8042          */
8043         hf = header_field_parse(p, params->field_names[0], &h0);
8044         mf = metadata_field_parse(p, params->field_names[0]);
8045         if (!hf && !mf)
8046                 return -EINVAL;
8047
8048         for (i = 1; i < params->n_fields; i++)
8049                 if (h0) {
8050                         struct header *h;
8051
8052                         hf = header_field_parse(p, params->field_names[i], &h);
8053                         if (!hf || (h->id != h0->id))
8054                                 return -EINVAL;
8055                 } else {
8056                         mf = metadata_field_parse(p, params->field_names[i]);
8057                         if (!mf)
8058                                 return -EINVAL;
8059                 }
8060
8061         /* Check that there are no duplicated match fields. */
8062         for (i = 0; i < params->n_fields; i++) {
8063                 const char *field_name = params->field_names[i];
8064                 uint32_t j;
8065
8066                 for (j = i + 1; j < params->n_fields; j++)
8067                         if (!strcmp(params->field_names[j], field_name))
8068                                 return -EINVAL;
8069         }
8070
8071         /* Return. */
8072         if (header)
8073                 *header = h0;
8074
8075         return 0;
8076 }
8077
8078 static int
8079 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8080 {
8081         struct struct_type *mst = p->metadata_st, *ast = a->st;
8082         struct field *mf, *af;
8083         uint32_t mf_pos, i;
8084
8085         if (!ast) {
8086                 if (mf_name)
8087                         return -EINVAL;
8088
8089                 return 0;
8090         }
8091
8092         /* Check that mf_name is the name of a valid meta-data field. */
8093         CHECK_NAME(mf_name, EINVAL);
8094         mf = metadata_field_parse(p, mf_name);
8095         CHECK(mf, EINVAL);
8096
8097         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8098          * all the action arguments.
8099          */
8100         mf_pos = mf - mst->fields;
8101         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8102
8103         /* Check that the size of each of the identified meta-data fields matches exactly the size
8104          * of the corresponding action argument.
8105          */
8106         for (i = 0; i < ast->n_fields; i++) {
8107                 mf = &mst->fields[mf_pos + i];
8108                 af = &ast->fields[i];
8109
8110                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8111         }
8112
8113         return 0;
8114 }
8115
8116 static int
8117 learner_action_learning_check(struct rte_swx_pipeline *p,
8118                               struct action *action,
8119                               const char **action_names,
8120                               uint32_t n_actions)
8121 {
8122         uint32_t i;
8123
8124         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8125          * the action passed as argument to the "learn" instruction) is also enabled for the
8126          * current learner table.
8127          */
8128         for (i = 0; i < action->n_instructions; i++) {
8129                 struct instruction *instr = &action->instructions[i];
8130                 uint32_t found = 0, j;
8131
8132                 if (instr->type != INSTR_LEARNER_LEARN)
8133                         continue;
8134
8135                 for (j = 0; j < n_actions; j++) {
8136                         struct action *a;
8137
8138                         a = action_find(p, action_names[j]);
8139                         if (!a)
8140                                 return -EINVAL;
8141
8142                         if (a->id == instr->learn.action_id)
8143                                 found = 1;
8144                 }
8145
8146                 if (!found)
8147                         return -EINVAL;
8148         }
8149
8150         return 0;
8151 }
8152
8153 int
8154 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8155                               const char *name,
8156                               struct rte_swx_pipeline_learner_params *params,
8157                               uint32_t size,
8158                               uint32_t timeout)
8159 {
8160         struct learner *l = NULL;
8161         struct action *default_action;
8162         struct header *header = NULL;
8163         uint32_t action_data_size_max = 0, i;
8164         int status = 0;
8165
8166         CHECK(p, EINVAL);
8167
8168         CHECK_NAME(name, EINVAL);
8169         CHECK(!table_find(p, name), EEXIST);
8170         CHECK(!selector_find(p, name), EEXIST);
8171         CHECK(!learner_find(p, name), EEXIST);
8172
8173         CHECK(params, EINVAL);
8174
8175         /* Match checks. */
8176         status = learner_match_fields_check(p, params, &header);
8177         if (status)
8178                 return status;
8179
8180         /* Action checks. */
8181         CHECK(params->n_actions, EINVAL);
8182
8183         CHECK(params->action_names, EINVAL);
8184         for (i = 0; i < params->n_actions; i++) {
8185                 const char *action_name = params->action_names[i];
8186                 struct action *a;
8187                 uint32_t action_data_size;
8188
8189                 CHECK_NAME(action_name, EINVAL);
8190
8191                 a = action_find(p, action_name);
8192                 CHECK(a, EINVAL);
8193
8194                 status = learner_action_learning_check(p,
8195                                                        a,
8196                                                        params->action_names,
8197                                                        params->n_actions);
8198                 if (status)
8199                         return status;
8200
8201                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8202                 if (action_data_size > action_data_size_max)
8203                         action_data_size_max = action_data_size;
8204         }
8205
8206         CHECK_NAME(params->default_action_name, EINVAL);
8207         for (i = 0; i < p->n_actions; i++)
8208                 if (!strcmp(params->action_names[i],
8209                             params->default_action_name))
8210                         break;
8211         CHECK(i < params->n_actions, EINVAL);
8212
8213         default_action = action_find(p, params->default_action_name);
8214         CHECK((default_action->st && params->default_action_data) ||
8215               !params->default_action_data, EINVAL);
8216
8217         /* Any other checks. */
8218         CHECK(size, EINVAL);
8219         CHECK(timeout, EINVAL);
8220
8221         /* Memory allocation. */
8222         l = calloc(1, sizeof(struct learner));
8223         if (!l)
8224                 goto nomem;
8225
8226         l->fields = calloc(params->n_fields, sizeof(struct field *));
8227         if (!l->fields)
8228                 goto nomem;
8229
8230         l->actions = calloc(params->n_actions, sizeof(struct action *));
8231         if (!l->actions)
8232                 goto nomem;
8233
8234         if (action_data_size_max) {
8235                 l->default_action_data = calloc(1, action_data_size_max);
8236                 if (!l->default_action_data)
8237                         goto nomem;
8238         }
8239
8240         /* Node initialization. */
8241         strcpy(l->name, name);
8242
8243         for (i = 0; i < params->n_fields; i++) {
8244                 const char *field_name = params->field_names[i];
8245
8246                 l->fields[i] = header ?
8247                         header_field_parse(p, field_name, NULL) :
8248                         metadata_field_parse(p, field_name);
8249         }
8250
8251         l->n_fields = params->n_fields;
8252
8253         l->header = header;
8254
8255         for (i = 0; i < params->n_actions; i++)
8256                 l->actions[i] = action_find(p, params->action_names[i]);
8257
8258         l->default_action = default_action;
8259
8260         if (default_action->st)
8261                 memcpy(l->default_action_data,
8262                        params->default_action_data,
8263                        default_action->st->n_bits / 8);
8264
8265         l->n_actions = params->n_actions;
8266
8267         l->default_action_is_const = params->default_action_is_const;
8268
8269         l->action_data_size_max = action_data_size_max;
8270
8271         l->size = size;
8272
8273         l->timeout = timeout;
8274
8275         l->id = p->n_learners;
8276
8277         /* Node add to tailq. */
8278         TAILQ_INSERT_TAIL(&p->learners, l, node);
8279         p->n_learners++;
8280
8281         return 0;
8282
8283 nomem:
8284         if (!l)
8285                 return -ENOMEM;
8286
8287         free(l->actions);
8288         free(l->fields);
8289         free(l);
8290
8291         return -ENOMEM;
8292 }
8293
8294 static void
8295 learner_params_free(struct rte_swx_table_learner_params *params)
8296 {
8297         if (!params)
8298                 return;
8299
8300         free(params->key_mask0);
8301
8302         free(params);
8303 }
8304
8305 static struct rte_swx_table_learner_params *
8306 learner_params_get(struct learner *l)
8307 {
8308         struct rte_swx_table_learner_params *params = NULL;
8309         struct field *first, *last;
8310         uint32_t i;
8311
8312         /* Memory allocation. */
8313         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8314         if (!params)
8315                 goto error;
8316
8317         /* Find first (smallest offset) and last (biggest offset) match fields. */
8318         first = l->fields[0];
8319         last = l->fields[0];
8320
8321         for (i = 0; i < l->n_fields; i++) {
8322                 struct field *f = l->fields[i];
8323
8324                 if (f->offset < first->offset)
8325                         first = f;
8326
8327                 if (f->offset > last->offset)
8328                         last = f;
8329         }
8330
8331         /* Key offset and size. */
8332         params->key_offset = first->offset / 8;
8333         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8334
8335         /* Memory allocation. */
8336         params->key_mask0 = calloc(1, params->key_size);
8337         if (!params->key_mask0)
8338                 goto error;
8339
8340         /* Key mask. */
8341         for (i = 0; i < l->n_fields; i++) {
8342                 struct field *f = l->fields[i];
8343                 uint32_t start = (f->offset - first->offset) / 8;
8344                 size_t size = f->n_bits / 8;
8345
8346                 memset(&params->key_mask0[start], 0xFF, size);
8347         }
8348
8349         /* Action data size. */
8350         params->action_data_size = l->action_data_size_max;
8351
8352         /* Maximum number of keys. */
8353         params->n_keys_max = l->size;
8354
8355         /* Timeout. */
8356         params->key_timeout = l->timeout;
8357
8358         return params;
8359
8360 error:
8361         learner_params_free(params);
8362         return NULL;
8363 }
8364
8365 static void
8366 learner_build_free(struct rte_swx_pipeline *p)
8367 {
8368         uint32_t i;
8369
8370         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8371                 struct thread *t = &p->threads[i];
8372                 uint32_t j;
8373
8374                 if (!t->learners)
8375                         continue;
8376
8377                 for (j = 0; j < p->n_learners; j++) {
8378                         struct learner_runtime *r = &t->learners[j];
8379
8380                         free(r->mailbox);
8381                 }
8382
8383                 free(t->learners);
8384                 t->learners = NULL;
8385         }
8386
8387         if (p->learner_stats) {
8388                 for (i = 0; i < p->n_learners; i++)
8389                         free(p->learner_stats[i].n_pkts_action);
8390
8391                 free(p->learner_stats);
8392         }
8393 }
8394
8395 static int
8396 learner_build(struct rte_swx_pipeline *p)
8397 {
8398         uint32_t i;
8399         int status = 0;
8400
8401         /* Per pipeline: learner statistics. */
8402         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8403         CHECK(p->learner_stats, ENOMEM);
8404
8405         for (i = 0; i < p->n_learners; i++) {
8406                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8407                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8408         }
8409
8410         /* Per thread: learner run-time. */
8411         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8412                 struct thread *t = &p->threads[i];
8413                 struct learner *l;
8414
8415                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8416                 if (!t->learners) {
8417                         status = -ENOMEM;
8418                         goto error;
8419                 }
8420
8421                 TAILQ_FOREACH(l, &p->learners, node) {
8422                         struct learner_runtime *r = &t->learners[l->id];
8423                         uint64_t size;
8424
8425                         /* r->mailbox. */
8426                         size = rte_swx_table_learner_mailbox_size_get();
8427                         if (size) {
8428                                 r->mailbox = calloc(1, size);
8429                                 if (!r->mailbox) {
8430                                         status = -ENOMEM;
8431                                         goto error;
8432                                 }
8433                         }
8434
8435                         /* r->key. */
8436                         r->key = l->header ?
8437                                 &t->structs[l->header->struct_id] :
8438                                 &t->structs[p->metadata_struct_id];
8439                 }
8440         }
8441
8442         return 0;
8443
8444 error:
8445         learner_build_free(p);
8446         return status;
8447 }
8448
8449 static void
8450 learner_free(struct rte_swx_pipeline *p)
8451 {
8452         learner_build_free(p);
8453
8454         /* Learner tables. */
8455         for ( ; ; ) {
8456                 struct learner *l;
8457
8458                 l = TAILQ_FIRST(&p->learners);
8459                 if (!l)
8460                         break;
8461
8462                 TAILQ_REMOVE(&p->learners, l, node);
8463                 free(l->fields);
8464                 free(l->actions);
8465                 free(l->default_action_data);
8466                 free(l);
8467         }
8468 }
8469
8470 /*
8471  * Table state.
8472  */
8473 static int
8474 table_state_build(struct rte_swx_pipeline *p)
8475 {
8476         struct table *table;
8477         struct selector *s;
8478         struct learner *l;
8479
8480         p->table_state = calloc(p->n_tables + p->n_selectors,
8481                                 sizeof(struct rte_swx_table_state));
8482         CHECK(p->table_state, ENOMEM);
8483
8484         TAILQ_FOREACH(table, &p->tables, node) {
8485                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8486
8487                 if (table->type) {
8488                         struct rte_swx_table_params *params;
8489
8490                         /* ts->obj. */
8491                         params = table_params_get(table);
8492                         CHECK(params, ENOMEM);
8493
8494                         ts->obj = table->type->ops.create(params,
8495                                 NULL,
8496                                 table->args,
8497                                 p->numa_node);
8498
8499                         table_params_free(params);
8500                         CHECK(ts->obj, ENODEV);
8501                 }
8502
8503                 /* ts->default_action_data. */
8504                 if (table->action_data_size_max) {
8505                         ts->default_action_data =
8506                                 malloc(table->action_data_size_max);
8507                         CHECK(ts->default_action_data, ENOMEM);
8508
8509                         memcpy(ts->default_action_data,
8510                                table->default_action_data,
8511                                table->action_data_size_max);
8512                 }
8513
8514                 /* ts->default_action_id. */
8515                 ts->default_action_id = table->default_action->id;
8516         }
8517
8518         TAILQ_FOREACH(s, &p->selectors, node) {
8519                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8520                 struct rte_swx_table_selector_params *params;
8521
8522                 /* ts->obj. */
8523                 params = selector_table_params_get(s);
8524                 CHECK(params, ENOMEM);
8525
8526                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8527
8528                 selector_params_free(params);
8529                 CHECK(ts->obj, ENODEV);
8530         }
8531
8532         TAILQ_FOREACH(l, &p->learners, node) {
8533                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8534                         p->n_selectors + l->id];
8535                 struct rte_swx_table_learner_params *params;
8536
8537                 /* ts->obj. */
8538                 params = learner_params_get(l);
8539                 CHECK(params, ENOMEM);
8540
8541                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8542                 learner_params_free(params);
8543                 CHECK(ts->obj, ENODEV);
8544
8545                 /* ts->default_action_data. */
8546                 if (l->action_data_size_max) {
8547                         ts->default_action_data = malloc(l->action_data_size_max);
8548                         CHECK(ts->default_action_data, ENOMEM);
8549
8550                         memcpy(ts->default_action_data,
8551                                l->default_action_data,
8552                                l->action_data_size_max);
8553                 }
8554
8555                 /* ts->default_action_id. */
8556                 ts->default_action_id = l->default_action->id;
8557         }
8558
8559         return 0;
8560 }
8561
8562 static void
8563 table_state_build_free(struct rte_swx_pipeline *p)
8564 {
8565         uint32_t i;
8566
8567         if (!p->table_state)
8568                 return;
8569
8570         for (i = 0; i < p->n_tables; i++) {
8571                 struct rte_swx_table_state *ts = &p->table_state[i];
8572                 struct table *table = table_find_by_id(p, i);
8573
8574                 /* ts->obj. */
8575                 if (table->type && ts->obj)
8576                         table->type->ops.free(ts->obj);
8577
8578                 /* ts->default_action_data. */
8579                 free(ts->default_action_data);
8580         }
8581
8582         for (i = 0; i < p->n_selectors; i++) {
8583                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8584
8585                 /* ts->obj. */
8586                 if (ts->obj)
8587                         rte_swx_table_selector_free(ts->obj);
8588         }
8589
8590         for (i = 0; i < p->n_learners; i++) {
8591                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8592
8593                 /* ts->obj. */
8594                 if (ts->obj)
8595                         rte_swx_table_learner_free(ts->obj);
8596
8597                 /* ts->default_action_data. */
8598                 free(ts->default_action_data);
8599         }
8600
8601         free(p->table_state);
8602         p->table_state = NULL;
8603 }
8604
8605 static void
8606 table_state_free(struct rte_swx_pipeline *p)
8607 {
8608         table_state_build_free(p);
8609 }
8610
8611 /*
8612  * Register array.
8613  */
8614 static struct regarray *
8615 regarray_find(struct rte_swx_pipeline *p, const char *name)
8616 {
8617         struct regarray *elem;
8618
8619         TAILQ_FOREACH(elem, &p->regarrays, node)
8620                 if (!strcmp(elem->name, name))
8621                         return elem;
8622
8623         return NULL;
8624 }
8625
8626 static struct regarray *
8627 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8628 {
8629         struct regarray *elem = NULL;
8630
8631         TAILQ_FOREACH(elem, &p->regarrays, node)
8632                 if (elem->id == id)
8633                         return elem;
8634
8635         return NULL;
8636 }
8637
8638 int
8639 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8640                               const char *name,
8641                               uint32_t size,
8642                               uint64_t init_val)
8643 {
8644         struct regarray *r;
8645
8646         CHECK(p, EINVAL);
8647
8648         CHECK_NAME(name, EINVAL);
8649         CHECK(!regarray_find(p, name), EEXIST);
8650
8651         CHECK(size, EINVAL);
8652         size = rte_align32pow2(size);
8653
8654         /* Memory allocation. */
8655         r = calloc(1, sizeof(struct regarray));
8656         CHECK(r, ENOMEM);
8657
8658         /* Node initialization. */
8659         strcpy(r->name, name);
8660         r->init_val = init_val;
8661         r->size = size;
8662         r->id = p->n_regarrays;
8663
8664         /* Node add to tailq. */
8665         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8666         p->n_regarrays++;
8667
8668         return 0;
8669 }
8670
8671 static int
8672 regarray_build(struct rte_swx_pipeline *p)
8673 {
8674         struct regarray *regarray;
8675
8676         if (!p->n_regarrays)
8677                 return 0;
8678
8679         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8680         CHECK(p->regarray_runtime, ENOMEM);
8681
8682         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8683                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8684                 uint32_t i;
8685
8686                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8687                                          RTE_CACHE_LINE_SIZE,
8688                                          p->numa_node);
8689                 CHECK(r->regarray, ENOMEM);
8690
8691                 if (regarray->init_val)
8692                         for (i = 0; i < regarray->size; i++)
8693                                 r->regarray[i] = regarray->init_val;
8694
8695                 r->size_mask = regarray->size - 1;
8696         }
8697
8698         return 0;
8699 }
8700
8701 static void
8702 regarray_build_free(struct rte_swx_pipeline *p)
8703 {
8704         uint32_t i;
8705
8706         if (!p->regarray_runtime)
8707                 return;
8708
8709         for (i = 0; i < p->n_regarrays; i++) {
8710                 struct regarray *regarray = regarray_find_by_id(p, i);
8711                 struct regarray_runtime *r = &p->regarray_runtime[i];
8712
8713                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8714         }
8715
8716         free(p->regarray_runtime);
8717         p->regarray_runtime = NULL;
8718 }
8719
8720 static void
8721 regarray_free(struct rte_swx_pipeline *p)
8722 {
8723         regarray_build_free(p);
8724
8725         for ( ; ; ) {
8726                 struct regarray *elem;
8727
8728                 elem = TAILQ_FIRST(&p->regarrays);
8729                 if (!elem)
8730                         break;
8731
8732                 TAILQ_REMOVE(&p->regarrays, elem, node);
8733                 free(elem);
8734         }
8735 }
8736
8737 /*
8738  * Meter array.
8739  */
8740 static struct meter_profile *
8741 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8742 {
8743         struct meter_profile *elem;
8744
8745         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8746                 if (!strcmp(elem->name, name))
8747                         return elem;
8748
8749         return NULL;
8750 }
8751
8752 static struct metarray *
8753 metarray_find(struct rte_swx_pipeline *p, const char *name)
8754 {
8755         struct metarray *elem;
8756
8757         TAILQ_FOREACH(elem, &p->metarrays, node)
8758                 if (!strcmp(elem->name, name))
8759                         return elem;
8760
8761         return NULL;
8762 }
8763
8764 static struct metarray *
8765 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8766 {
8767         struct metarray *elem = NULL;
8768
8769         TAILQ_FOREACH(elem, &p->metarrays, node)
8770                 if (elem->id == id)
8771                         return elem;
8772
8773         return NULL;
8774 }
8775
8776 int
8777 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8778                                  const char *name,
8779                                  uint32_t size)
8780 {
8781         struct metarray *m;
8782
8783         CHECK(p, EINVAL);
8784
8785         CHECK_NAME(name, EINVAL);
8786         CHECK(!metarray_find(p, name), EEXIST);
8787
8788         CHECK(size, EINVAL);
8789         size = rte_align32pow2(size);
8790
8791         /* Memory allocation. */
8792         m = calloc(1, sizeof(struct metarray));
8793         CHECK(m, ENOMEM);
8794
8795         /* Node initialization. */
8796         strcpy(m->name, name);
8797         m->size = size;
8798         m->id = p->n_metarrays;
8799
8800         /* Node add to tailq. */
8801         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8802         p->n_metarrays++;
8803
8804         return 0;
8805 }
8806
8807 struct meter_profile meter_profile_default = {
8808         .node = {0},
8809         .name = "",
8810         .params = {0},
8811
8812         .profile = {
8813                 .cbs = 10000,
8814                 .pbs = 10000,
8815                 .cir_period = 1,
8816                 .cir_bytes_per_period = 1,
8817                 .pir_period = 1,
8818                 .pir_bytes_per_period = 1,
8819         },
8820
8821         .n_users = 0,
8822 };
8823
8824 static void
8825 meter_init(struct meter *m)
8826 {
8827         memset(m, 0, sizeof(struct meter));
8828         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8829         m->profile = &meter_profile_default;
8830         m->color_mask = RTE_COLOR_GREEN;
8831
8832         meter_profile_default.n_users++;
8833 }
8834
8835 static int
8836 metarray_build(struct rte_swx_pipeline *p)
8837 {
8838         struct metarray *m;
8839
8840         if (!p->n_metarrays)
8841                 return 0;
8842
8843         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8844         CHECK(p->metarray_runtime, ENOMEM);
8845
8846         TAILQ_FOREACH(m, &p->metarrays, node) {
8847                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8848                 uint32_t i;
8849
8850                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8851                                          RTE_CACHE_LINE_SIZE,
8852                                          p->numa_node);
8853                 CHECK(r->metarray, ENOMEM);
8854
8855                 for (i = 0; i < m->size; i++)
8856                         meter_init(&r->metarray[i]);
8857
8858                 r->size_mask = m->size - 1;
8859         }
8860
8861         return 0;
8862 }
8863
8864 static void
8865 metarray_build_free(struct rte_swx_pipeline *p)
8866 {
8867         uint32_t i;
8868
8869         if (!p->metarray_runtime)
8870                 return;
8871
8872         for (i = 0; i < p->n_metarrays; i++) {
8873                 struct metarray *m = metarray_find_by_id(p, i);
8874                 struct metarray_runtime *r = &p->metarray_runtime[i];
8875
8876                 env_free(r->metarray, m->size * sizeof(struct meter));
8877         }
8878
8879         free(p->metarray_runtime);
8880         p->metarray_runtime = NULL;
8881 }
8882
8883 static void
8884 metarray_free(struct rte_swx_pipeline *p)
8885 {
8886         metarray_build_free(p);
8887
8888         /* Meter arrays. */
8889         for ( ; ; ) {
8890                 struct metarray *elem;
8891
8892                 elem = TAILQ_FIRST(&p->metarrays);
8893                 if (!elem)
8894                         break;
8895
8896                 TAILQ_REMOVE(&p->metarrays, elem, node);
8897                 free(elem);
8898         }
8899
8900         /* Meter profiles. */
8901         for ( ; ; ) {
8902                 struct meter_profile *elem;
8903
8904                 elem = TAILQ_FIRST(&p->meter_profiles);
8905                 if (!elem)
8906                         break;
8907
8908                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8909                 free(elem);
8910         }
8911 }
8912
8913 /*
8914  * Pipeline.
8915  */
8916 int
8917 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8918 {
8919         struct rte_swx_pipeline *pipeline;
8920
8921         /* Check input parameters. */
8922         CHECK(p, EINVAL);
8923
8924         /* Memory allocation. */
8925         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8926         CHECK(pipeline, ENOMEM);
8927
8928         /* Initialization. */
8929         TAILQ_INIT(&pipeline->struct_types);
8930         TAILQ_INIT(&pipeline->port_in_types);
8931         TAILQ_INIT(&pipeline->ports_in);
8932         TAILQ_INIT(&pipeline->port_out_types);
8933         TAILQ_INIT(&pipeline->ports_out);
8934         TAILQ_INIT(&pipeline->extern_types);
8935         TAILQ_INIT(&pipeline->extern_objs);
8936         TAILQ_INIT(&pipeline->extern_funcs);
8937         TAILQ_INIT(&pipeline->headers);
8938         TAILQ_INIT(&pipeline->actions);
8939         TAILQ_INIT(&pipeline->table_types);
8940         TAILQ_INIT(&pipeline->tables);
8941         TAILQ_INIT(&pipeline->selectors);
8942         TAILQ_INIT(&pipeline->learners);
8943         TAILQ_INIT(&pipeline->regarrays);
8944         TAILQ_INIT(&pipeline->meter_profiles);
8945         TAILQ_INIT(&pipeline->metarrays);
8946
8947         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
8948         pipeline->numa_node = numa_node;
8949
8950         *p = pipeline;
8951         return 0;
8952 }
8953
8954 void
8955 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8956 {
8957         void *lib;
8958
8959         if (!p)
8960                 return;
8961
8962         lib = p->lib;
8963
8964         free(p->instruction_data);
8965         free(p->instructions);
8966
8967         metarray_free(p);
8968         regarray_free(p);
8969         table_state_free(p);
8970         learner_free(p);
8971         selector_free(p);
8972         table_free(p);
8973         action_free(p);
8974         instruction_table_free(p);
8975         metadata_free(p);
8976         header_free(p);
8977         extern_func_free(p);
8978         extern_obj_free(p);
8979         port_out_free(p);
8980         port_in_free(p);
8981         struct_free(p);
8982
8983         free(p);
8984
8985         if (lib)
8986                 dlclose(lib);
8987 }
8988
8989 int
8990 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
8991                                      const char **instructions,
8992                                      uint32_t n_instructions)
8993 {
8994         int err;
8995         uint32_t i;
8996
8997         err = instruction_config(p, NULL, instructions, n_instructions);
8998         if (err)
8999                 return err;
9000
9001         /* Thread instruction pointer reset. */
9002         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9003                 struct thread *t = &p->threads[i];
9004
9005                 thread_ip_reset(p, t);
9006         }
9007
9008         return 0;
9009 }
9010
9011 static int
9012 pipeline_compile(struct rte_swx_pipeline *p);
9013
9014 int
9015 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9016 {
9017         int status;
9018
9019         CHECK(p, EINVAL);
9020         CHECK(p->build_done == 0, EEXIST);
9021
9022         status = port_in_build(p);
9023         if (status)
9024                 goto error;
9025
9026         status = port_out_build(p);
9027         if (status)
9028                 goto error;
9029
9030         status = struct_build(p);
9031         if (status)
9032                 goto error;
9033
9034         status = extern_obj_build(p);
9035         if (status)
9036                 goto error;
9037
9038         status = extern_func_build(p);
9039         if (status)
9040                 goto error;
9041
9042         status = header_build(p);
9043         if (status)
9044                 goto error;
9045
9046         status = metadata_build(p);
9047         if (status)
9048                 goto error;
9049
9050         status = instruction_table_build(p);
9051         if (status)
9052                 goto error;
9053
9054         status = action_build(p);
9055         if (status)
9056                 goto error;
9057
9058         status = table_build(p);
9059         if (status)
9060                 goto error;
9061
9062         status = selector_build(p);
9063         if (status)
9064                 goto error;
9065
9066         status = learner_build(p);
9067         if (status)
9068                 goto error;
9069
9070         status = table_state_build(p);
9071         if (status)
9072                 goto error;
9073
9074         status = regarray_build(p);
9075         if (status)
9076                 goto error;
9077
9078         status = metarray_build(p);
9079         if (status)
9080                 goto error;
9081
9082         p->build_done = 1;
9083
9084         pipeline_compile(p);
9085
9086         return 0;
9087
9088 error:
9089         metarray_build_free(p);
9090         regarray_build_free(p);
9091         table_state_build_free(p);
9092         learner_build_free(p);
9093         selector_build_free(p);
9094         table_build_free(p);
9095         action_build_free(p);
9096         instruction_table_build_free(p);
9097         metadata_build_free(p);
9098         header_build_free(p);
9099         extern_func_build_free(p);
9100         extern_obj_build_free(p);
9101         port_out_build_free(p);
9102         port_in_build_free(p);
9103         struct_build_free(p);
9104
9105         return status;
9106 }
9107
9108 void
9109 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9110 {
9111         uint32_t i;
9112
9113         for (i = 0; i < n_instructions; i++)
9114                 instr_exec(p);
9115 }
9116
9117 void
9118 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9119 {
9120         uint32_t i;
9121
9122         for (i = 0; i < p->n_ports_out; i++) {
9123                 struct port_out_runtime *port = &p->out[i];
9124
9125                 if (port->flush)
9126                         port->flush(port->obj);
9127         }
9128 }
9129
9130 /*
9131  * Control.
9132  */
9133 int
9134 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9135                               struct rte_swx_ctl_pipeline_info *pipeline)
9136 {
9137         struct action *action;
9138         struct table *table;
9139         uint32_t n_actions = 0, n_tables = 0;
9140
9141         if (!p || !pipeline)
9142                 return -EINVAL;
9143
9144         TAILQ_FOREACH(action, &p->actions, node)
9145                 n_actions++;
9146
9147         TAILQ_FOREACH(table, &p->tables, node)
9148                 n_tables++;
9149
9150         pipeline->n_ports_in = p->n_ports_in;
9151         pipeline->n_ports_out = p->n_ports_out;
9152         pipeline->n_actions = n_actions;
9153         pipeline->n_tables = n_tables;
9154         pipeline->n_selectors = p->n_selectors;
9155         pipeline->n_learners = p->n_learners;
9156         pipeline->n_regarrays = p->n_regarrays;
9157         pipeline->n_metarrays = p->n_metarrays;
9158
9159         return 0;
9160 }
9161
9162 int
9163 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9164 {
9165         if (!p || !numa_node)
9166                 return -EINVAL;
9167
9168         *numa_node = p->numa_node;
9169         return 0;
9170 }
9171
9172 int
9173 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9174                             uint32_t action_id,
9175                             struct rte_swx_ctl_action_info *action)
9176 {
9177         struct action *a = NULL;
9178
9179         if (!p || (action_id >= p->n_actions) || !action)
9180                 return -EINVAL;
9181
9182         a = action_find_by_id(p, action_id);
9183         if (!a)
9184                 return -EINVAL;
9185
9186         strcpy(action->name, a->name);
9187         action->n_args = a->st ? a->st->n_fields : 0;
9188         return 0;
9189 }
9190
9191 int
9192 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9193                                 uint32_t action_id,
9194                                 uint32_t action_arg_id,
9195                                 struct rte_swx_ctl_action_arg_info *action_arg)
9196 {
9197         struct action *a = NULL;
9198         struct field *arg = NULL;
9199
9200         if (!p || (action_id >= p->n_actions) || !action_arg)
9201                 return -EINVAL;
9202
9203         a = action_find_by_id(p, action_id);
9204         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9205                 return -EINVAL;
9206
9207         arg = &a->st->fields[action_arg_id];
9208         strcpy(action_arg->name, arg->name);
9209         action_arg->n_bits = arg->n_bits;
9210         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9211
9212         return 0;
9213 }
9214
9215 int
9216 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9217                            uint32_t table_id,
9218                            struct rte_swx_ctl_table_info *table)
9219 {
9220         struct table *t = NULL;
9221
9222         if (!p || !table)
9223                 return -EINVAL;
9224
9225         t = table_find_by_id(p, table_id);
9226         if (!t)
9227                 return -EINVAL;
9228
9229         strcpy(table->name, t->name);
9230         strcpy(table->args, t->args);
9231         table->n_match_fields = t->n_fields;
9232         table->n_actions = t->n_actions;
9233         table->default_action_is_const = t->default_action_is_const;
9234         table->size = t->size;
9235         return 0;
9236 }
9237
9238 int
9239 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9240         uint32_t table_id,
9241         uint32_t match_field_id,
9242         struct rte_swx_ctl_table_match_field_info *match_field)
9243 {
9244         struct table *t;
9245         struct match_field *f;
9246
9247         if (!p || (table_id >= p->n_tables) || !match_field)
9248                 return -EINVAL;
9249
9250         t = table_find_by_id(p, table_id);
9251         if (!t || (match_field_id >= t->n_fields))
9252                 return -EINVAL;
9253
9254         f = &t->fields[match_field_id];
9255         match_field->match_type = f->match_type;
9256         match_field->is_header = t->header ? 1 : 0;
9257         match_field->n_bits = f->field->n_bits;
9258         match_field->offset = f->field->offset;
9259
9260         return 0;
9261 }
9262
9263 int
9264 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9265         uint32_t table_id,
9266         uint32_t table_action_id,
9267         struct rte_swx_ctl_table_action_info *table_action)
9268 {
9269         struct table *t;
9270
9271         if (!p || (table_id >= p->n_tables) || !table_action)
9272                 return -EINVAL;
9273
9274         t = table_find_by_id(p, table_id);
9275         if (!t || (table_action_id >= t->n_actions))
9276                 return -EINVAL;
9277
9278         table_action->action_id = t->actions[table_action_id]->id;
9279
9280         return 0;
9281 }
9282
9283 int
9284 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9285                           uint32_t table_id,
9286                           struct rte_swx_table_ops *table_ops,
9287                           int *is_stub)
9288 {
9289         struct table *t;
9290
9291         if (!p || (table_id >= p->n_tables))
9292                 return -EINVAL;
9293
9294         t = table_find_by_id(p, table_id);
9295         if (!t)
9296                 return -EINVAL;
9297
9298         if (t->type) {
9299                 if (table_ops)
9300                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9301                 *is_stub = 0;
9302         } else {
9303                 *is_stub = 1;
9304         }
9305
9306         return 0;
9307 }
9308
9309 int
9310 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9311                               uint32_t selector_id,
9312                               struct rte_swx_ctl_selector_info *selector)
9313 {
9314         struct selector *s = NULL;
9315
9316         if (!p || !selector)
9317                 return -EINVAL;
9318
9319         s = selector_find_by_id(p, selector_id);
9320         if (!s)
9321                 return -EINVAL;
9322
9323         strcpy(selector->name, s->name);
9324
9325         selector->n_selector_fields = s->n_selector_fields;
9326         selector->n_groups_max = s->n_groups_max;
9327         selector->n_members_per_group_max = s->n_members_per_group_max;
9328
9329         return 0;
9330 }
9331
9332 int
9333 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9334          uint32_t selector_id,
9335          struct rte_swx_ctl_table_match_field_info *field)
9336 {
9337         struct selector *s;
9338
9339         if (!p || (selector_id >= p->n_selectors) || !field)
9340                 return -EINVAL;
9341
9342         s = selector_find_by_id(p, selector_id);
9343         if (!s)
9344                 return -EINVAL;
9345
9346         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9347         field->is_header = 0;
9348         field->n_bits = s->group_id_field->n_bits;
9349         field->offset = s->group_id_field->offset;
9350
9351         return 0;
9352 }
9353
9354 int
9355 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9356          uint32_t selector_id,
9357          uint32_t selector_field_id,
9358          struct rte_swx_ctl_table_match_field_info *field)
9359 {
9360         struct selector *s;
9361         struct field *f;
9362
9363         if (!p || (selector_id >= p->n_selectors) || !field)
9364                 return -EINVAL;
9365
9366         s = selector_find_by_id(p, selector_id);
9367         if (!s || (selector_field_id >= s->n_selector_fields))
9368                 return -EINVAL;
9369
9370         f = s->selector_fields[selector_field_id];
9371         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9372         field->is_header = s->selector_header ? 1 : 0;
9373         field->n_bits = f->n_bits;
9374         field->offset = f->offset;
9375
9376         return 0;
9377 }
9378
9379 int
9380 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9381          uint32_t selector_id,
9382          struct rte_swx_ctl_table_match_field_info *field)
9383 {
9384         struct selector *s;
9385
9386         if (!p || (selector_id >= p->n_selectors) || !field)
9387                 return -EINVAL;
9388
9389         s = selector_find_by_id(p, selector_id);
9390         if (!s)
9391                 return -EINVAL;
9392
9393         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9394         field->is_header = 0;
9395         field->n_bits = s->member_id_field->n_bits;
9396         field->offset = s->member_id_field->offset;
9397
9398         return 0;
9399 }
9400
9401 int
9402 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9403                              uint32_t learner_id,
9404                              struct rte_swx_ctl_learner_info *learner)
9405 {
9406         struct learner *l = NULL;
9407
9408         if (!p || !learner)
9409                 return -EINVAL;
9410
9411         l = learner_find_by_id(p, learner_id);
9412         if (!l)
9413                 return -EINVAL;
9414
9415         strcpy(learner->name, l->name);
9416
9417         learner->n_match_fields = l->n_fields;
9418         learner->n_actions = l->n_actions;
9419         learner->default_action_is_const = l->default_action_is_const;
9420         learner->size = l->size;
9421
9422         return 0;
9423 }
9424
9425 int
9426 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9427                                          uint32_t learner_id,
9428                                          uint32_t match_field_id,
9429                                          struct rte_swx_ctl_table_match_field_info *match_field)
9430 {
9431         struct learner *l;
9432         struct field *f;
9433
9434         if (!p || (learner_id >= p->n_learners) || !match_field)
9435                 return -EINVAL;
9436
9437         l = learner_find_by_id(p, learner_id);
9438         if (!l || (match_field_id >= l->n_fields))
9439                 return -EINVAL;
9440
9441         f = l->fields[match_field_id];
9442         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9443         match_field->is_header = l->header ? 1 : 0;
9444         match_field->n_bits = f->n_bits;
9445         match_field->offset = f->offset;
9446
9447         return 0;
9448 }
9449
9450 int
9451 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9452                                     uint32_t learner_id,
9453                                     uint32_t learner_action_id,
9454                                     struct rte_swx_ctl_table_action_info *learner_action)
9455 {
9456         struct learner *l;
9457
9458         if (!p || (learner_id >= p->n_learners) || !learner_action)
9459                 return -EINVAL;
9460
9461         l = learner_find_by_id(p, learner_id);
9462         if (!l || (learner_action_id >= l->n_actions))
9463                 return -EINVAL;
9464
9465         learner_action->action_id = l->actions[learner_action_id]->id;
9466
9467         return 0;
9468 }
9469
9470 int
9471 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9472                                  struct rte_swx_table_state **table_state)
9473 {
9474         if (!p || !table_state || !p->build_done)
9475                 return -EINVAL;
9476
9477         *table_state = p->table_state;
9478         return 0;
9479 }
9480
9481 int
9482 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9483                                  struct rte_swx_table_state *table_state)
9484 {
9485         if (!p || !table_state || !p->build_done)
9486                 return -EINVAL;
9487
9488         p->table_state = table_state;
9489         return 0;
9490 }
9491
9492 int
9493 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9494                                         uint32_t port_id,
9495                                         struct rte_swx_port_in_stats *stats)
9496 {
9497         struct port_in *port;
9498
9499         if (!p || !stats)
9500                 return -EINVAL;
9501
9502         port = port_in_find(p, port_id);
9503         if (!port)
9504                 return -EINVAL;
9505
9506         port->type->ops.stats_read(port->obj, stats);
9507         return 0;
9508 }
9509
9510 int
9511 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9512                                          uint32_t port_id,
9513                                          struct rte_swx_port_out_stats *stats)
9514 {
9515         struct port_out *port;
9516
9517         if (!p || !stats)
9518                 return -EINVAL;
9519
9520         port = port_out_find(p, port_id);
9521         if (!port)
9522                 return -EINVAL;
9523
9524         port->type->ops.stats_read(port->obj, stats);
9525         return 0;
9526 }
9527
9528 int
9529 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9530                                       const char *table_name,
9531                                       struct rte_swx_table_stats *stats)
9532 {
9533         struct table *table;
9534         struct table_statistics *table_stats;
9535
9536         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9537                 return -EINVAL;
9538
9539         table = table_find(p, table_name);
9540         if (!table)
9541                 return -EINVAL;
9542
9543         table_stats = &p->table_stats[table->id];
9544
9545         memcpy(stats->n_pkts_action,
9546                table_stats->n_pkts_action,
9547                p->n_actions * sizeof(uint64_t));
9548
9549         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9550         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9551
9552         return 0;
9553 }
9554
9555 int
9556 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9557         const char *selector_name,
9558         struct rte_swx_pipeline_selector_stats *stats)
9559 {
9560         struct selector *s;
9561
9562         if (!p || !selector_name || !selector_name[0] || !stats)
9563                 return -EINVAL;
9564
9565         s = selector_find(p, selector_name);
9566         if (!s)
9567                 return -EINVAL;
9568
9569         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9570
9571         return 0;
9572 }
9573
9574 int
9575 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9576                                         const char *learner_name,
9577                                         struct rte_swx_learner_stats *stats)
9578 {
9579         struct learner *l;
9580         struct learner_statistics *learner_stats;
9581
9582         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9583                 return -EINVAL;
9584
9585         l = learner_find(p, learner_name);
9586         if (!l)
9587                 return -EINVAL;
9588
9589         learner_stats = &p->learner_stats[l->id];
9590
9591         memcpy(stats->n_pkts_action,
9592                learner_stats->n_pkts_action,
9593                p->n_actions * sizeof(uint64_t));
9594
9595         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9596         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9597
9598         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9599         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9600
9601         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9602
9603         return 0;
9604 }
9605
9606 int
9607 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9608                               uint32_t regarray_id,
9609                               struct rte_swx_ctl_regarray_info *regarray)
9610 {
9611         struct regarray *r;
9612
9613         if (!p || !regarray)
9614                 return -EINVAL;
9615
9616         r = regarray_find_by_id(p, regarray_id);
9617         if (!r)
9618                 return -EINVAL;
9619
9620         strcpy(regarray->name, r->name);
9621         regarray->size = r->size;
9622         return 0;
9623 }
9624
9625 int
9626 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9627                                    const char *regarray_name,
9628                                    uint32_t regarray_index,
9629                                    uint64_t *value)
9630 {
9631         struct regarray *regarray;
9632         struct regarray_runtime *r;
9633
9634         if (!p || !regarray_name || !value)
9635                 return -EINVAL;
9636
9637         regarray = regarray_find(p, regarray_name);
9638         if (!regarray || (regarray_index >= regarray->size))
9639                 return -EINVAL;
9640
9641         r = &p->regarray_runtime[regarray->id];
9642         *value = r->regarray[regarray_index];
9643         return 0;
9644 }
9645
9646 int
9647 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9648                                    const char *regarray_name,
9649                                    uint32_t regarray_index,
9650                                    uint64_t value)
9651 {
9652         struct regarray *regarray;
9653         struct regarray_runtime *r;
9654
9655         if (!p || !regarray_name)
9656                 return -EINVAL;
9657
9658         regarray = regarray_find(p, regarray_name);
9659         if (!regarray || (regarray_index >= regarray->size))
9660                 return -EINVAL;
9661
9662         r = &p->regarray_runtime[regarray->id];
9663         r->regarray[regarray_index] = value;
9664         return 0;
9665 }
9666
9667 int
9668 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9669                               uint32_t metarray_id,
9670                               struct rte_swx_ctl_metarray_info *metarray)
9671 {
9672         struct metarray *m;
9673
9674         if (!p || !metarray)
9675                 return -EINVAL;
9676
9677         m = metarray_find_by_id(p, metarray_id);
9678         if (!m)
9679                 return -EINVAL;
9680
9681         strcpy(metarray->name, m->name);
9682         metarray->size = m->size;
9683         return 0;
9684 }
9685
9686 int
9687 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9688                               const char *name,
9689                               struct rte_meter_trtcm_params *params)
9690 {
9691         struct meter_profile *mp;
9692         int status;
9693
9694         CHECK(p, EINVAL);
9695         CHECK_NAME(name, EINVAL);
9696         CHECK(params, EINVAL);
9697         CHECK(!meter_profile_find(p, name), EEXIST);
9698
9699         /* Node allocation. */
9700         mp = calloc(1, sizeof(struct meter_profile));
9701         CHECK(mp, ENOMEM);
9702
9703         /* Node initialization. */
9704         strcpy(mp->name, name);
9705         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9706         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9707         if (status) {
9708                 free(mp);
9709                 CHECK(0, EINVAL);
9710         }
9711
9712         /* Node add to tailq. */
9713         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9714
9715         return 0;
9716 }
9717
9718 int
9719 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9720                                  const char *name)
9721 {
9722         struct meter_profile *mp;
9723
9724         CHECK(p, EINVAL);
9725         CHECK_NAME(name, EINVAL);
9726
9727         mp = meter_profile_find(p, name);
9728         CHECK(mp, EINVAL);
9729         CHECK(!mp->n_users, EBUSY);
9730
9731         /* Remove node from tailq. */
9732         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9733         free(mp);
9734
9735         return 0;
9736 }
9737
9738 int
9739 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9740                         const char *metarray_name,
9741                         uint32_t metarray_index)
9742 {
9743         struct meter_profile *mp_old;
9744         struct metarray *metarray;
9745         struct metarray_runtime *metarray_runtime;
9746         struct meter *m;
9747
9748         CHECK(p, EINVAL);
9749         CHECK_NAME(metarray_name, EINVAL);
9750
9751         metarray = metarray_find(p, metarray_name);
9752         CHECK(metarray, EINVAL);
9753         CHECK(metarray_index < metarray->size, EINVAL);
9754
9755         metarray_runtime = &p->metarray_runtime[metarray->id];
9756         m = &metarray_runtime->metarray[metarray_index];
9757         mp_old = m->profile;
9758
9759         meter_init(m);
9760
9761         mp_old->n_users--;
9762
9763         return 0;
9764 }
9765
9766 int
9767 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9768                       const char *metarray_name,
9769                       uint32_t metarray_index,
9770                       const char *profile_name)
9771 {
9772         struct meter_profile *mp, *mp_old;
9773         struct metarray *metarray;
9774         struct metarray_runtime *metarray_runtime;
9775         struct meter *m;
9776
9777         CHECK(p, EINVAL);
9778         CHECK_NAME(metarray_name, EINVAL);
9779
9780         metarray = metarray_find(p, metarray_name);
9781         CHECK(metarray, EINVAL);
9782         CHECK(metarray_index < metarray->size, EINVAL);
9783
9784         mp = meter_profile_find(p, profile_name);
9785         CHECK(mp, EINVAL);
9786
9787         metarray_runtime = &p->metarray_runtime[metarray->id];
9788         m = &metarray_runtime->metarray[metarray_index];
9789         mp_old = m->profile;
9790
9791         memset(m, 0, sizeof(struct meter));
9792         rte_meter_trtcm_config(&m->m, &mp->profile);
9793         m->profile = mp;
9794         m->color_mask = RTE_COLORS;
9795
9796         mp->n_users++;
9797         mp_old->n_users--;
9798
9799         return 0;
9800 }
9801
9802 int
9803 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9804                              const char *metarray_name,
9805                              uint32_t metarray_index,
9806                              struct rte_swx_ctl_meter_stats *stats)
9807 {
9808         struct metarray *metarray;
9809         struct metarray_runtime *metarray_runtime;
9810         struct meter *m;
9811
9812         CHECK(p, EINVAL);
9813         CHECK_NAME(metarray_name, EINVAL);
9814
9815         metarray = metarray_find(p, metarray_name);
9816         CHECK(metarray, EINVAL);
9817         CHECK(metarray_index < metarray->size, EINVAL);
9818
9819         CHECK(stats, EINVAL);
9820
9821         metarray_runtime = &p->metarray_runtime[metarray->id];
9822         m = &metarray_runtime->metarray[metarray_index];
9823
9824         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9825         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9826
9827         return 0;
9828 }
9829
9830 /*
9831  * Pipeline compilation.
9832  */
9833 static const char *
9834 instr_type_to_name(struct instruction *instr)
9835 {
9836         switch (instr->type) {
9837         case INSTR_RX: return "INSTR_RX";
9838
9839         case INSTR_TX: return "INSTR_TX";
9840         case INSTR_TX_I: return "INSTR_TX_I";
9841
9842         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
9843         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
9844         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
9845         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
9846         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
9847         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
9848         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
9849         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
9850
9851         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
9852
9853         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
9854
9855         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
9856         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
9857         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
9858         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
9859         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
9860         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
9861         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
9862         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
9863         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
9864
9865         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
9866         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
9867
9868         case INSTR_MOV: return "INSTR_MOV";
9869         case INSTR_MOV_MH: return "INSTR_MOV_MH";
9870         case INSTR_MOV_HM: return "INSTR_MOV_HM";
9871         case INSTR_MOV_HH: return "INSTR_MOV_HH";
9872         case INSTR_MOV_I: return "INSTR_MOV_I";
9873
9874         case INSTR_DMA_HT: return "INSTR_DMA_HT";
9875         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
9876         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
9877         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
9878         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
9879         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
9880         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
9881         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
9882
9883         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
9884         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
9885         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
9886         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
9887         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
9888         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
9889
9890         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
9891         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
9892         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
9893         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
9894         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
9895         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
9896
9897         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
9898         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
9899         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
9900         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
9901
9902         case INSTR_ALU_AND: return "INSTR_ALU_AND";
9903         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
9904         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
9905         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
9906         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
9907
9908         case INSTR_ALU_OR: return "INSTR_ALU_OR";
9909         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
9910         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
9911         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
9912         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
9913
9914         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
9915         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
9916         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
9917         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
9918         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
9919
9920         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
9921         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
9922         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
9923         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
9924         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
9925         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
9926
9927         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
9928         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
9929         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
9930         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
9931         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
9932         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
9933
9934         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
9935         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
9936         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
9937
9938         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
9939         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
9940         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
9941         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
9942         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
9943         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
9944
9945         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
9946         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
9947         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
9948         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
9949         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
9950         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
9951         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
9952         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
9953         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
9954
9955         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
9956         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
9957         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
9958         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
9959         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
9960         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
9961         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
9962         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
9963         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
9964
9965         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
9966         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
9967         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
9968
9969         case INSTR_METER_HHM: return "INSTR_METER_HHM";
9970         case INSTR_METER_HHI: return "INSTR_METER_HHI";
9971         case INSTR_METER_HMM: return "INSTR_METER_HMM";
9972         case INSTR_METER_HMI: return "INSTR_METER_HMI";
9973         case INSTR_METER_MHM: return "INSTR_METER_MHM";
9974         case INSTR_METER_MHI: return "INSTR_METER_MHI";
9975         case INSTR_METER_MMM: return "INSTR_METER_MMM";
9976         case INSTR_METER_MMI: return "INSTR_METER_MMI";
9977         case INSTR_METER_IHM: return "INSTR_METER_IHM";
9978         case INSTR_METER_IHI: return "INSTR_METER_IHI";
9979         case INSTR_METER_IMM: return "INSTR_METER_IMM";
9980         case INSTR_METER_IMI: return "INSTR_METER_IMI";
9981
9982         case INSTR_TABLE: return "INSTR_TABLE";
9983         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
9984         case INSTR_SELECTOR: return "INSTR_SELECTOR";
9985         case INSTR_LEARNER: return "INSTR_LEARNER";
9986         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
9987
9988         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
9989         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
9990
9991         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
9992         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
9993
9994         case INSTR_JMP: return "INSTR_JMP";
9995         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
9996         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
9997         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
9998         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
9999         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10000         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10001         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10002         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10003         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10004         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10005         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10006         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10007         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10008         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10009         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10010         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10011         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10012         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10013         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10014         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10015         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10016         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10017         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10018         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10019         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10020         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10021         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10022         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10023
10024         case INSTR_RETURN: return "INSTR_RETURN";
10025
10026         default: return "INSTR_UNKNOWN";
10027         }
10028 }
10029
10030 typedef void
10031 (*instruction_export_t)(struct instruction *, FILE *);
10032
10033 static void
10034 instr_io_export(struct instruction *instr, FILE *f)
10035 {
10036         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10037
10038         /* n_io, n_io_imm, n_hdrs. */
10039         if (instr->type == INSTR_RX ||
10040             instr->type == INSTR_TX ||
10041             instr->type == INSTR_HDR_EXTRACT_M ||
10042             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10043                 n_io = 1;
10044
10045         if (instr->type == INSTR_TX_I)
10046                 n_io_imm = 1;
10047
10048         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10049                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10050
10051         if (instr->type == INSTR_HDR_EXTRACT_M ||
10052             instr->type == INSTR_HDR_LOOKAHEAD ||
10053             instr->type == INSTR_HDR_EMIT)
10054                 n_hdrs = 1;
10055
10056         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10057                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10058
10059         /* instr. */
10060         fprintf(f,
10061                 "\t{\n"
10062                 "\t\t.type = %s,\n",
10063                 instr_type_to_name(instr));
10064
10065         /* instr.io. */
10066         fprintf(f,
10067                 "\t\t.io = {\n");
10068
10069         /* instr.io.io. */
10070         if (n_io)
10071                 fprintf(f,
10072                         "\t\t\t.io = {\n"
10073                         "\t\t\t\t.offset = %u,\n"
10074                         "\t\t\t\t.n_bits = %u,\n"
10075                         "\t\t\t},\n",
10076                         instr->io.io.offset,
10077                         instr->io.io.n_bits);
10078
10079         if (n_io_imm)
10080                 fprintf(f,
10081                         "\t\t\t.io = {\n"
10082                         "\t\t\t\t.val = %u,\n"
10083                         "\t\t\t},\n",
10084                         instr->io.io.val);
10085
10086         /* instr.io.hdr. */
10087         if (n_hdrs) {
10088                 fprintf(f,
10089                         "\t\t.hdr = {\n");
10090
10091                 /* instr.io.hdr.header_id. */
10092                 fprintf(f,
10093                         "\t\t\t.header_id = {");
10094
10095                 for (i = 0; i < n_hdrs; i++)
10096                         fprintf(f,
10097                                 "%u, ",
10098                                 instr->io.hdr.header_id[i]);
10099
10100                 fprintf(f,
10101                         "},\n");
10102
10103                 /* instr.io.hdr.struct_id. */
10104                 fprintf(f,
10105                         "\t\t\t.struct_id = {");
10106
10107                 for (i = 0; i < n_hdrs; i++)
10108                         fprintf(f,
10109                                 "%u, ",
10110                                 instr->io.hdr.struct_id[i]);
10111
10112                 fprintf(f,
10113                         "},\n");
10114
10115                 /* instr.io.hdr.n_bytes. */
10116                 fprintf(f,
10117                         "\t\t\t.n_bytes = {");
10118
10119                 for (i = 0; i < n_hdrs; i++)
10120                         fprintf(f,
10121                                 "%u, ",
10122                                 instr->io.hdr.n_bytes[i]);
10123
10124                 fprintf(f,
10125                         "},\n");
10126
10127                 /* instr.io.hdr - closing curly brace. */
10128                 fprintf(f,
10129                         "\t\t\t}\n,");
10130         }
10131
10132         /* instr.io - closing curly brace. */
10133         fprintf(f,
10134                 "\t\t},\n");
10135
10136         /* instr - closing curly brace. */
10137         fprintf(f,
10138                 "\t},\n");
10139 }
10140
10141 static void
10142 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10143 {
10144         fprintf(f,
10145                 "\t{\n"
10146                 "\t\t.type = %s,\n"
10147                 "\t\t.valid = {\n"
10148                 "\t\t\t.header_id = %u,\n"
10149                 "\t\t},\n"
10150                 "\t},\n",
10151                 instr_type_to_name(instr),
10152                 instr->valid.header_id);
10153 }
10154
10155 static void
10156 instr_mov_export(struct instruction *instr, FILE *f)
10157 {
10158         if (instr->type != INSTR_MOV_I)
10159                 fprintf(f,
10160                         "\t{\n"
10161                         "\t\t.type = %s,\n"
10162                         "\t\t.mov = {\n"
10163                         "\t\t\t.dst = {\n"
10164                         "\t\t\t\t.struct_id = %u,\n"
10165                         "\t\t\t\t.n_bits = %u,\n"
10166                         "\t\t\t\t.offset = %u,\n"
10167                         "\t\t\t},\n"
10168                         "\t\t\t.src = {\n"
10169                         "\t\t\t\t.struct_id = %u,\n"
10170                         "\t\t\t\t.n_bits = %u,\n"
10171                         "\t\t\t\t.offset = %u,\n"
10172                         "\t\t\t},\n"
10173                         "\t\t},\n"
10174                         "\t},\n",
10175                         instr_type_to_name(instr),
10176                         instr->mov.dst.struct_id,
10177                         instr->mov.dst.n_bits,
10178                         instr->mov.dst.offset,
10179                         instr->mov.src.struct_id,
10180                         instr->mov.src.n_bits,
10181                         instr->mov.src.offset);
10182         else
10183                 fprintf(f,
10184                         "\t{\n"
10185                         "\t\t.type = %s,\n"
10186                         "\t\t.mov = {\n"
10187                         "\t\t\t.dst = {\n"
10188                         "\t\t\t\t.struct_id = %u,\n"
10189                         "\t\t\t\t.n_bits = %u,\n"
10190                         "\t\t\t\t.offset = %u,\n"
10191                         "\t\t\t}\n,"
10192                         "\t\t\t.src_val = %" PRIu64 ",\n"
10193                         "\t\t},\n"
10194                         "\t},\n",
10195                         instr_type_to_name(instr),
10196                         instr->mov.dst.struct_id,
10197                         instr->mov.dst.n_bits,
10198                         instr->mov.dst.offset,
10199                         instr->mov.src_val);
10200 }
10201
10202 static void
10203 instr_dma_ht_export(struct instruction *instr, FILE *f)
10204 {
10205         uint32_t n_dma = 0, i;
10206
10207         /* n_dma. */
10208         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10209
10210         /* instr. */
10211         fprintf(f,
10212                 "\t{\n"
10213                 "\t\t.type = %s,\n",
10214                 instr_type_to_name(instr));
10215
10216         /* instr.dma. */
10217         fprintf(f,
10218                 "\t\t.dma = {\n");
10219
10220         /* instr.dma.dst. */
10221         fprintf(f,
10222                 "\t\t\t.dst = {\n");
10223
10224         /* instr.dma.dst.header_id. */
10225         fprintf(f,
10226                 "\t\t\t\t.header_id = {");
10227
10228         for (i = 0; i < n_dma; i++)
10229                 fprintf(f,
10230                         "%u, ",
10231                         instr->dma.dst.header_id[i]);
10232
10233         fprintf(f,
10234                 "},\n");
10235
10236         /* instr.dma.dst.struct_id. */
10237         fprintf(f,
10238                 "\t\t\t\t.struct_id = {");
10239
10240         for (i = 0; i < n_dma; i++)
10241                 fprintf(f,
10242                         "%u, ",
10243                         instr->dma.dst.struct_id[i]);
10244
10245         fprintf(f,
10246                 "},\n");
10247
10248         /* instr.dma.dst - closing curly brace. */
10249         fprintf(f,
10250                 "\t\t\t},\n");
10251
10252         /* instr.dma.src. */
10253         fprintf(f,
10254                 "\t\t\t.src = {\n");
10255
10256         /* instr.dma.src.offset. */
10257         fprintf(f,
10258                 "\t\t\t\t.offset = {");
10259
10260         for (i = 0; i < n_dma; i++)
10261                 fprintf(f,
10262                         "%u, ",
10263                         instr->dma.src.offset[i]);
10264
10265         fprintf(f,
10266                 "},\n");
10267
10268         /* instr.dma.src - closing curly brace. */
10269         fprintf(f,
10270                 "\t\t\t},\n");
10271
10272         /* instr.dma.n_bytes. */
10273         fprintf(f,
10274                 "\t\t\t.n_bytes = {");
10275
10276         for (i = 0; i < n_dma; i++)
10277                 fprintf(f,
10278                         "%u, ",
10279                         instr->dma.n_bytes[i]);
10280
10281         fprintf(f,
10282                 "},\n");
10283
10284         /* instr.dma - closing curly brace. */
10285         fprintf(f,
10286                 "\t\t},\n");
10287
10288         /* instr - closing curly brace. */
10289         fprintf(f,
10290                 "\t},\n");
10291 }
10292
10293 static void
10294 instr_alu_export(struct instruction *instr, FILE *f)
10295 {
10296         int imm = 0;
10297
10298         if (instr->type == INSTR_ALU_ADD_MI ||
10299             instr->type == INSTR_ALU_ADD_HI ||
10300             instr->type == INSTR_ALU_SUB_MI ||
10301             instr->type == INSTR_ALU_SUB_HI ||
10302             instr->type == INSTR_ALU_SHL_MI ||
10303             instr->type == INSTR_ALU_SHL_HI ||
10304             instr->type == INSTR_ALU_SHR_MI ||
10305             instr->type == INSTR_ALU_SHR_HI ||
10306             instr->type == INSTR_ALU_AND_I ||
10307             instr->type == INSTR_ALU_OR_I ||
10308             instr->type == INSTR_ALU_XOR_I)
10309                 imm = 1;
10310
10311         if (!imm)
10312                 fprintf(f,
10313                         "\t{\n"
10314                         "\t\t.type = %s,\n"
10315                         "\t\t.alu = {\n"
10316                         "\t\t\t.dst = {\n"
10317                         "\t\t\t\t.struct_id = %u,\n"
10318                         "\t\t\t\t.n_bits = %u,\n"
10319                         "\t\t\t\t.offset = %u,\n"
10320                         "\t\t\t},\n"
10321                         "\t\t\t.src = {\n"
10322                         "\t\t\t\t.struct_id = %u,\n"
10323                         "\t\t\t\t.n_bits = %u,\n"
10324                         "\t\t\t\t.offset = %u,\n"
10325                         "\t\t\t},\n"
10326                         "\t\t},\n"
10327                         "\t},\n",
10328                         instr_type_to_name(instr),
10329                         instr->alu.dst.struct_id,
10330                         instr->alu.dst.n_bits,
10331                         instr->alu.dst.offset,
10332                         instr->alu.src.struct_id,
10333                         instr->alu.src.n_bits,
10334                         instr->alu.src.offset);
10335         else
10336                 fprintf(f,
10337                         "\t{\n"
10338                         "\t\t.type = %s,\n"
10339                         "\t\t.alu = {\n"
10340                         "\t\t\t.dst = {\n"
10341                         "\t\t\t\t.struct_id = %u,\n"
10342                         "\t\t\t\t.n_bits = %u,\n"
10343                         "\t\t\t\t.offset = %u,\n"
10344                         "\t\t\t}\n,"
10345                         "\t\t\t.src_val = %" PRIu64 ",\n"
10346                         "\t\t},\n"
10347                         "\t},\n",
10348                         instr_type_to_name(instr),
10349                         instr->alu.dst.struct_id,
10350                         instr->alu.dst.n_bits,
10351                         instr->alu.dst.offset,
10352                         instr->alu.src_val);
10353 }
10354
10355 static void
10356 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10357 {
10358         int prefetch  = 0, idx_imm = 0, src_imm = 0;
10359
10360         if (instr->type == INSTR_REGPREFETCH_RH ||
10361             instr->type == INSTR_REGPREFETCH_RM ||
10362             instr->type == INSTR_REGPREFETCH_RI)
10363                 prefetch = 1;
10364
10365         /* index is the 3rd operand for the regrd instruction and the 2nd
10366          * operand for the regwr and regadd instructions.
10367          */
10368         if (instr->type == INSTR_REGPREFETCH_RI ||
10369             instr->type == INSTR_REGRD_HRI ||
10370             instr->type == INSTR_REGRD_MRI ||
10371             instr->type == INSTR_REGWR_RIH ||
10372             instr->type == INSTR_REGWR_RIM ||
10373             instr->type == INSTR_REGWR_RII ||
10374             instr->type == INSTR_REGADD_RIH ||
10375             instr->type == INSTR_REGADD_RIM ||
10376             instr->type == INSTR_REGADD_RII)
10377                 idx_imm = 1;
10378
10379         /* src is the 3rd operand for the regwr and regadd instructions. */
10380         if (instr->type == INSTR_REGWR_RHI ||
10381             instr->type == INSTR_REGWR_RMI ||
10382             instr->type == INSTR_REGWR_RII ||
10383             instr->type == INSTR_REGADD_RHI ||
10384             instr->type == INSTR_REGADD_RMI ||
10385             instr->type == INSTR_REGADD_RII)
10386                 src_imm = 1;
10387
10388         /* instr.regarray.regarray_id. */
10389         fprintf(f,
10390                 "\t{\n"
10391                 "\t\t.type = %s,\n"
10392                 "\t\t.regarray = {\n"
10393                 "\t\t\t.regarray_id = %u,\n",
10394                 instr_type_to_name(instr),
10395                 instr->regarray.regarray_id);
10396
10397         /* instr.regarray.idx / instr.regarray.idx_val. */
10398         if (!idx_imm)
10399                 fprintf(f,
10400                         "\t\t\t\t.idx = {\n"
10401                         "\t\t\t\t\t.struct_id = %u,\n"
10402                         "\t\t\t\t\t.n_bits = %u,\n"
10403                         "\t\t\t\t\t.offset = %u,\n"
10404                         "\t\t\t\t},\n",
10405                         instr->regarray.idx.struct_id,
10406                         instr->regarray.idx.n_bits,
10407                         instr->regarray.idx.offset);
10408         else
10409                 fprintf(f,
10410                         "\t\t\t\t.idx_val = %u,\n",
10411                         instr->regarray.idx_val);
10412
10413         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10414         if (!prefetch) {
10415                 if (!src_imm)
10416                         fprintf(f,
10417                                 "\t\t\t\t.dstsrc = {\n"
10418                                 "\t\t\t\t\t.struct_id = %u,\n"
10419                                 "\t\t\t\t\t.n_bits = %u,\n"
10420                                 "\t\t\t\t\t.offset = %u,\n"
10421                                 "\t\t\t\t},\n",
10422                                 instr->regarray.dstsrc.struct_id,
10423                                 instr->regarray.dstsrc.n_bits,
10424                                 instr->regarray.dstsrc.offset);
10425                 else
10426                         fprintf(f,
10427                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10428                                 instr->regarray.dstsrc_val);
10429         }
10430
10431         /* instr.regarray and instr - closing curly braces. */
10432         fprintf(f,
10433                 "\t\t},\n"
10434                 "\t},\n");
10435 }
10436
10437 static void
10438 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10439 {
10440         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
10441
10442         if (instr->type == INSTR_METPREFETCH_H ||
10443             instr->type == INSTR_METPREFETCH_M ||
10444             instr->type == INSTR_METPREFETCH_I)
10445                 prefetch = 1;
10446
10447         /* idx_imm. */
10448         if (instr->type == INSTR_METPREFETCH_I ||
10449             instr->type == INSTR_METER_IHM ||
10450             instr->type == INSTR_METER_IHI ||
10451             instr->type == INSTR_METER_IMM ||
10452             instr->type == INSTR_METER_IMI)
10453                 idx_imm = 1;
10454
10455         /* color_in_imm. */
10456         if (instr->type == INSTR_METER_HHI ||
10457             instr->type == INSTR_METER_HMI ||
10458             instr->type == INSTR_METER_MHI ||
10459             instr->type == INSTR_METER_MMI ||
10460             instr->type == INSTR_METER_IHI ||
10461             instr->type == INSTR_METER_IMI)
10462                 color_in_imm = 1;
10463
10464         /* instr.meter.metarray_id. */
10465         fprintf(f,
10466                 "\t{\n"
10467                 "\t\t.type = %s,\n"
10468                 "\t\t.meter = {\n"
10469                 "\t\t\t.metarray_id = %u,\n",
10470                 instr_type_to_name(instr),
10471                 instr->meter.metarray_id);
10472
10473         /* instr.meter.idx / instr.meter.idx_val. */
10474         if (!idx_imm)
10475                 fprintf(f,
10476                         "\t\t\t.idx = {\n"
10477                         "\t\t\t\t.struct_id = %u,\n"
10478                         "\t\t\t\t.n_bits = %u,\n"
10479                         "\t\t\t\t.offset = %u,\n"
10480                         "\t\t\t},\n",
10481                         instr->meter.idx.struct_id,
10482                         instr->meter.idx.n_bits,
10483                         instr->meter.idx.offset);
10484         else
10485                 fprintf(f,
10486                         "\t\t\t.idx_val = %u,\n",
10487                         instr->meter.idx_val);
10488
10489         if (!prefetch) {
10490                 /* instr.meter.length. */
10491                 fprintf(f,
10492                         "\t\t\t.length = {\n"
10493                         "\t\t\t\t.struct_id = %u,\n"
10494                         "\t\t\t\t.n_bits = %u,\n"
10495                         "\t\t\t\t.offset = %u,\n"
10496                         "\t\t\t},\n",
10497                         instr->meter.length.struct_id,
10498                         instr->meter.length.n_bits,
10499                         instr->meter.length.offset);
10500
10501                 /* instr.meter.color_in / instr.meter.color_in_val. */
10502                 if (!color_in_imm)
10503                         fprintf(f,
10504                                 "\t\t\t.color_in = {\n"
10505                                 "\t\t\t\t.struct_id = %u,\n"
10506                                 "\t\t\t\t.n_bits = %u,\n"
10507                                 "\t\t\t\t.offset = %u,\n"
10508                                 "\t\t\t},\n",
10509                                 instr->meter.color_in.struct_id,
10510                                 instr->meter.color_in.n_bits,
10511                                 instr->meter.color_in.offset);
10512                 else
10513                         fprintf(f,
10514                                 "\t\t\t.color_in_val = %u,\n",
10515                                 (uint32_t)instr->meter.color_in_val);
10516
10517                 /* instr.meter.color_out. */
10518                 fprintf(f,
10519                         "\t\t\t.color_out = {\n"
10520                         "\t\t\t\t.struct_id = %u,\n"
10521                         "\t\t\t\t.n_bits = %u,\n"
10522                         "\t\t\t\t.offset = %u,\n"
10523                         "\t\t\t},\n",
10524                         instr->meter.color_out.struct_id,
10525                         instr->meter.color_out.n_bits,
10526                         instr->meter.color_out.offset);
10527         }
10528
10529         /* instr.meter and instr - closing curly braces. */
10530         fprintf(f,
10531                 "\t\t},\n"
10532                 "\t},\n");
10533 }
10534
10535 static void
10536 instr_table_export(struct instruction *instr,
10537                 FILE *f)
10538 {
10539         fprintf(f,
10540                 "\t{\n"
10541                 "\t\t.type = %s,\n"
10542                 "\t\t.table = {\n"
10543                 "\t\t\t.table_id = %u,\n"
10544                 "\t\t},\n"
10545                 "\t},\n",
10546                 instr_type_to_name(instr),
10547                 instr->table.table_id);
10548 }
10549
10550 static void
10551 instr_learn_export(struct instruction *instr, FILE *f)
10552 {
10553         fprintf(f,
10554                 "\t{\n"
10555                 "\t\t.type = %s,\n"
10556                 "\t\t.learn = {\n"
10557                 "\t\t\t\t.action_id = %u,\n"
10558                 "\t\t},\n"
10559                 "\t},\n",
10560                 instr_type_to_name(instr),
10561                 instr->learn.action_id);
10562 }
10563
10564 static void
10565 instr_forget_export(struct instruction *instr, FILE *f)
10566 {
10567         fprintf(f,
10568                 "\t{\n"
10569                 "\t\t.type = %s,\n"
10570                 "\t},\n",
10571                 instr_type_to_name(instr));
10572 }
10573
10574 static void
10575 instr_extern_export(struct instruction *instr, FILE *f)
10576 {
10577         if (instr->type == INSTR_EXTERN_OBJ)
10578                 fprintf(f,
10579                         "\t{\n"
10580                         "\t\t.type = %s,\n"
10581                         "\t\t.ext_obj = {\n"
10582                         "\t\t\t.ext_obj_id = %u,\n"
10583                         "\t\t\t.func_id = %u,\n"
10584                         "\t\t},\n"
10585                         "\t},\n",
10586                         instr_type_to_name(instr),
10587                         instr->ext_obj.ext_obj_id,
10588                         instr->ext_obj.func_id);
10589         else
10590                 fprintf(f,
10591                         "\t{\n"
10592                         "\t\t.type = %s,\n"
10593                         "\t\t.ext_func = {\n"
10594                         "\t\t\t.ext_func_id = %u,\n"
10595                         "\t\t},\n"
10596                         "\t},\n",
10597                         instr_type_to_name(instr),
10598                         instr->ext_func.ext_func_id);
10599 }
10600
10601 static void
10602 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10603 {
10604         fprintf(f,
10605                 "\t{\n"
10606                 "\t\t.type = %s,\n"
10607                 "\t\t.jmp = {\n"
10608                 "\t\t\t.ip = NULL,\n",
10609                 instr_type_to_name(instr));
10610
10611         switch (instr->type) {
10612         case INSTR_JMP_VALID:
10613         case INSTR_JMP_INVALID:
10614                 fprintf(f,
10615                         "\t\t\t.header_id = %u,\n",
10616                         instr->jmp.header_id);
10617                 break;
10618
10619         case INSTR_JMP_ACTION_HIT:
10620         case INSTR_JMP_ACTION_MISS:
10621                 fprintf(f,
10622                         "\t\t\t.action_id = %u,\n",
10623                         instr->jmp.action_id);
10624                 break;
10625
10626         case INSTR_JMP_EQ:
10627         case INSTR_JMP_EQ_MH:
10628         case INSTR_JMP_EQ_HM:
10629         case INSTR_JMP_EQ_HH:
10630         case INSTR_JMP_NEQ:
10631         case INSTR_JMP_NEQ_MH:
10632         case INSTR_JMP_NEQ_HM:
10633         case INSTR_JMP_NEQ_HH:
10634         case INSTR_JMP_LT:
10635         case INSTR_JMP_LT_MH:
10636         case INSTR_JMP_LT_HM:
10637         case INSTR_JMP_LT_HH:
10638         case INSTR_JMP_GT:
10639         case INSTR_JMP_GT_MH:
10640         case INSTR_JMP_GT_HM:
10641         case INSTR_JMP_GT_HH:
10642                 fprintf(f,
10643                         "\t\t\t.a = {\n"
10644                         "\t\t\t\t.struct_id = %u,\n"
10645                         "\t\t\t\t.n_bits = %u,\n"
10646                         "\t\t\t\t.offset = %u,\n"
10647                         "\t\t\t},\n"
10648                         "\t\t\t.b = {\n"
10649                         "\t\t\t\t.struct_id = %u,\n"
10650                         "\t\t\t\t.n_bits = %u,\n"
10651                         "\t\t\t\t.offset = %u,\n"
10652                         "\t\t\t},\n",
10653                         instr->jmp.a.struct_id,
10654                         instr->jmp.a.n_bits,
10655                         instr->jmp.a.offset,
10656                         instr->jmp.b.struct_id,
10657                         instr->jmp.b.n_bits,
10658                         instr->jmp.b.offset);
10659                 break;
10660
10661         case INSTR_JMP_EQ_I:
10662         case INSTR_JMP_NEQ_I:
10663         case INSTR_JMP_LT_MI:
10664         case INSTR_JMP_LT_HI:
10665         case INSTR_JMP_GT_MI:
10666         case INSTR_JMP_GT_HI:
10667                 fprintf(f,
10668                         "\t\t\t.a = {\n"
10669                         "\t\t\t\t.struct_id = %u,\n"
10670                         "\t\t\t\t.n_bits = %u,\n"
10671                         "\t\t\t\t.offset = %u,\n"
10672                         "\t\t\t}\n,"
10673                         "\t\t\t.b_val = %" PRIu64 ",\n",
10674                         instr->jmp.a.struct_id,
10675                         instr->jmp.a.n_bits,
10676                         instr->jmp.a.offset,
10677                         instr->jmp.b_val);
10678                 break;
10679
10680         default:
10681                 break;
10682         }
10683
10684         fprintf(f,
10685                 "\t\t},\n"
10686                 "\t},\n");
10687 }
10688
10689 static void
10690 instr_return_export(struct instruction *instr,
10691                 FILE *f)
10692 {
10693         fprintf(f,
10694                 "\t{\n"
10695                 "\t\t.type = %s,\n",
10696                 instr_type_to_name(instr));
10697
10698         fprintf(f,
10699                 "\t},\n");
10700 }
10701
10702 static instruction_export_t export_table[] = {
10703         [INSTR_RX] = instr_io_export,
10704
10705         [INSTR_TX] = instr_io_export,
10706         [INSTR_TX_I] = instr_io_export,
10707
10708         [INSTR_HDR_EXTRACT] = instr_io_export,
10709         [INSTR_HDR_EXTRACT2] = instr_io_export,
10710         [INSTR_HDR_EXTRACT3] = instr_io_export,
10711         [INSTR_HDR_EXTRACT4] = instr_io_export,
10712         [INSTR_HDR_EXTRACT5] = instr_io_export,
10713         [INSTR_HDR_EXTRACT6] = instr_io_export,
10714         [INSTR_HDR_EXTRACT7] = instr_io_export,
10715         [INSTR_HDR_EXTRACT8] = instr_io_export,
10716
10717         [INSTR_HDR_EXTRACT_M] = instr_io_export,
10718
10719         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10720
10721         [INSTR_HDR_EMIT] = instr_io_export,
10722         [INSTR_HDR_EMIT_TX] = instr_io_export,
10723         [INSTR_HDR_EMIT2_TX] = instr_io_export,
10724         [INSTR_HDR_EMIT3_TX] = instr_io_export,
10725         [INSTR_HDR_EMIT4_TX] = instr_io_export,
10726         [INSTR_HDR_EMIT5_TX] = instr_io_export,
10727         [INSTR_HDR_EMIT6_TX] = instr_io_export,
10728         [INSTR_HDR_EMIT7_TX] = instr_io_export,
10729         [INSTR_HDR_EMIT8_TX] = instr_io_export,
10730
10731         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10732         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10733
10734         [INSTR_MOV] = instr_mov_export,
10735         [INSTR_MOV_MH] = instr_mov_export,
10736         [INSTR_MOV_HM] = instr_mov_export,
10737         [INSTR_MOV_HH] = instr_mov_export,
10738         [INSTR_MOV_I] = instr_mov_export,
10739
10740         [INSTR_DMA_HT]  = instr_dma_ht_export,
10741         [INSTR_DMA_HT2] = instr_dma_ht_export,
10742         [INSTR_DMA_HT3] = instr_dma_ht_export,
10743         [INSTR_DMA_HT4] = instr_dma_ht_export,
10744         [INSTR_DMA_HT5] = instr_dma_ht_export,
10745         [INSTR_DMA_HT6] = instr_dma_ht_export,
10746         [INSTR_DMA_HT7] = instr_dma_ht_export,
10747         [INSTR_DMA_HT8] = instr_dma_ht_export,
10748
10749         [INSTR_ALU_ADD] = instr_alu_export,
10750         [INSTR_ALU_ADD_MH] = instr_alu_export,
10751         [INSTR_ALU_ADD_HM] = instr_alu_export,
10752         [INSTR_ALU_ADD_HH] = instr_alu_export,
10753         [INSTR_ALU_ADD_MI] = instr_alu_export,
10754         [INSTR_ALU_ADD_HI] = instr_alu_export,
10755
10756         [INSTR_ALU_SUB] = instr_alu_export,
10757         [INSTR_ALU_SUB_MH] = instr_alu_export,
10758         [INSTR_ALU_SUB_HM] = instr_alu_export,
10759         [INSTR_ALU_SUB_HH] = instr_alu_export,
10760         [INSTR_ALU_SUB_MI] = instr_alu_export,
10761         [INSTR_ALU_SUB_HI] = instr_alu_export,
10762
10763         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10764         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10765         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10766         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10767
10768         [INSTR_ALU_AND] = instr_alu_export,
10769         [INSTR_ALU_AND_MH] = instr_alu_export,
10770         [INSTR_ALU_AND_HM] = instr_alu_export,
10771         [INSTR_ALU_AND_HH] = instr_alu_export,
10772         [INSTR_ALU_AND_I] = instr_alu_export,
10773
10774         [INSTR_ALU_OR] = instr_alu_export,
10775         [INSTR_ALU_OR_MH] = instr_alu_export,
10776         [INSTR_ALU_OR_HM] = instr_alu_export,
10777         [INSTR_ALU_OR_HH] = instr_alu_export,
10778         [INSTR_ALU_OR_I] = instr_alu_export,
10779
10780         [INSTR_ALU_XOR] = instr_alu_export,
10781         [INSTR_ALU_XOR_MH] = instr_alu_export,
10782         [INSTR_ALU_XOR_HM] = instr_alu_export,
10783         [INSTR_ALU_XOR_HH] = instr_alu_export,
10784         [INSTR_ALU_XOR_I] = instr_alu_export,
10785
10786         [INSTR_ALU_SHL] = instr_alu_export,
10787         [INSTR_ALU_SHL_MH] = instr_alu_export,
10788         [INSTR_ALU_SHL_HM] = instr_alu_export,
10789         [INSTR_ALU_SHL_HH] = instr_alu_export,
10790         [INSTR_ALU_SHL_MI] = instr_alu_export,
10791         [INSTR_ALU_SHL_HI] = instr_alu_export,
10792
10793         [INSTR_ALU_SHR] = instr_alu_export,
10794         [INSTR_ALU_SHR_MH] = instr_alu_export,
10795         [INSTR_ALU_SHR_HM] = instr_alu_export,
10796         [INSTR_ALU_SHR_HH] = instr_alu_export,
10797         [INSTR_ALU_SHR_MI] = instr_alu_export,
10798         [INSTR_ALU_SHR_HI] = instr_alu_export,
10799
10800         [INSTR_REGPREFETCH_RH] = instr_reg_export,
10801         [INSTR_REGPREFETCH_RM] = instr_reg_export,
10802         [INSTR_REGPREFETCH_RI] = instr_reg_export,
10803
10804         [INSTR_REGRD_HRH] = instr_reg_export,
10805         [INSTR_REGRD_HRM] = instr_reg_export,
10806         [INSTR_REGRD_MRH] = instr_reg_export,
10807         [INSTR_REGRD_MRM] = instr_reg_export,
10808         [INSTR_REGRD_HRI] = instr_reg_export,
10809         [INSTR_REGRD_MRI] = instr_reg_export,
10810
10811         [INSTR_REGWR_RHH] = instr_reg_export,
10812         [INSTR_REGWR_RHM] = instr_reg_export,
10813         [INSTR_REGWR_RMH] = instr_reg_export,
10814         [INSTR_REGWR_RMM] = instr_reg_export,
10815         [INSTR_REGWR_RHI] = instr_reg_export,
10816         [INSTR_REGWR_RMI] = instr_reg_export,
10817         [INSTR_REGWR_RIH] = instr_reg_export,
10818         [INSTR_REGWR_RIM] = instr_reg_export,
10819         [INSTR_REGWR_RII] = instr_reg_export,
10820
10821         [INSTR_REGADD_RHH] = instr_reg_export,
10822         [INSTR_REGADD_RHM] = instr_reg_export,
10823         [INSTR_REGADD_RMH] = instr_reg_export,
10824         [INSTR_REGADD_RMM] = instr_reg_export,
10825         [INSTR_REGADD_RHI] = instr_reg_export,
10826         [INSTR_REGADD_RMI] = instr_reg_export,
10827         [INSTR_REGADD_RIH] = instr_reg_export,
10828         [INSTR_REGADD_RIM] = instr_reg_export,
10829         [INSTR_REGADD_RII] = instr_reg_export,
10830
10831         [INSTR_METPREFETCH_H] = instr_meter_export,
10832         [INSTR_METPREFETCH_M] = instr_meter_export,
10833         [INSTR_METPREFETCH_I] = instr_meter_export,
10834
10835         [INSTR_METER_HHM] = instr_meter_export,
10836         [INSTR_METER_HHI] = instr_meter_export,
10837         [INSTR_METER_HMM] = instr_meter_export,
10838         [INSTR_METER_HMI] = instr_meter_export,
10839         [INSTR_METER_MHM] = instr_meter_export,
10840         [INSTR_METER_MHI] = instr_meter_export,
10841         [INSTR_METER_MMM] = instr_meter_export,
10842         [INSTR_METER_MMI] = instr_meter_export,
10843         [INSTR_METER_IHM] = instr_meter_export,
10844         [INSTR_METER_IHI] = instr_meter_export,
10845         [INSTR_METER_IMM] = instr_meter_export,
10846         [INSTR_METER_IMI] = instr_meter_export,
10847
10848         [INSTR_TABLE] = instr_table_export,
10849         [INSTR_TABLE_AF] = instr_table_export,
10850         [INSTR_SELECTOR] = instr_table_export,
10851         [INSTR_LEARNER] = instr_table_export,
10852         [INSTR_LEARNER_AF] = instr_table_export,
10853
10854         [INSTR_LEARNER_LEARN] = instr_learn_export,
10855         [INSTR_LEARNER_FORGET] = instr_forget_export,
10856
10857         [INSTR_EXTERN_OBJ] = instr_extern_export,
10858         [INSTR_EXTERN_FUNC] = instr_extern_export,
10859
10860         [INSTR_JMP] = instr_jmp_export,
10861         [INSTR_JMP_VALID] = instr_jmp_export,
10862         [INSTR_JMP_INVALID] = instr_jmp_export,
10863         [INSTR_JMP_HIT] = instr_jmp_export,
10864         [INSTR_JMP_MISS] = instr_jmp_export,
10865         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
10866         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
10867
10868         [INSTR_JMP_EQ] = instr_jmp_export,
10869         [INSTR_JMP_EQ_MH] = instr_jmp_export,
10870         [INSTR_JMP_EQ_HM] = instr_jmp_export,
10871         [INSTR_JMP_EQ_HH] = instr_jmp_export,
10872         [INSTR_JMP_EQ_I] = instr_jmp_export,
10873
10874         [INSTR_JMP_NEQ] = instr_jmp_export,
10875         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
10876         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
10877         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
10878         [INSTR_JMP_NEQ_I] = instr_jmp_export,
10879
10880         [INSTR_JMP_LT] = instr_jmp_export,
10881         [INSTR_JMP_LT_MH] = instr_jmp_export,
10882         [INSTR_JMP_LT_HM] = instr_jmp_export,
10883         [INSTR_JMP_LT_HH] = instr_jmp_export,
10884         [INSTR_JMP_LT_MI] = instr_jmp_export,
10885         [INSTR_JMP_LT_HI] = instr_jmp_export,
10886
10887         [INSTR_JMP_GT] = instr_jmp_export,
10888         [INSTR_JMP_GT_MH] = instr_jmp_export,
10889         [INSTR_JMP_GT_HM] = instr_jmp_export,
10890         [INSTR_JMP_GT_HH] = instr_jmp_export,
10891         [INSTR_JMP_GT_MI] = instr_jmp_export,
10892         [INSTR_JMP_GT_HI] = instr_jmp_export,
10893
10894         [INSTR_RETURN] = instr_return_export,
10895 };
10896
10897 static void
10898 action_data_codegen(struct action *a, FILE *f)
10899 {
10900         uint32_t i;
10901
10902         fprintf(f,
10903                 "static const struct instruction action_%s_instructions[] = {\n",
10904                 a->name);
10905
10906         for (i = 0; i < a->n_instructions; i++) {
10907                 struct instruction *instr = &a->instructions[i];
10908                 instruction_export_t func = export_table[instr->type];
10909
10910                 func(instr, f);
10911         }
10912
10913         fprintf(f, "};\n");
10914 }
10915
10916 static const char *
10917 instr_type_to_func(struct instruction *instr)
10918 {
10919         switch (instr->type) {
10920         case INSTR_RX: return NULL;
10921
10922         case INSTR_TX: return "__instr_tx_exec";
10923         case INSTR_TX_I: return "__instr_tx_i_exec";
10924
10925         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
10926         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
10927         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
10928         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
10929         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
10930         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
10931         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
10932         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
10933
10934         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
10935
10936         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
10937
10938         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
10939         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
10940         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
10941         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
10942         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
10943         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
10944         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
10945         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
10946         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
10947
10948         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
10949         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
10950
10951         case INSTR_MOV: return "__instr_mov_exec";
10952         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
10953         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
10954         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
10955         case INSTR_MOV_I: return "__instr_mov_i_exec";
10956
10957         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
10958         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
10959         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
10960         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
10961         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
10962         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
10963         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
10964         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
10965
10966         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
10967         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
10968         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
10969         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
10970         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
10971         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
10972
10973         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
10974         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
10975         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
10976         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
10977         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
10978         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
10979
10980         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
10981         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
10982         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
10983         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
10984
10985         case INSTR_ALU_AND: return "__instr_alu_and_exec";
10986         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
10987         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
10988         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
10989         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
10990
10991         case INSTR_ALU_OR: return "__instr_alu_or_exec";
10992         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
10993         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
10994         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
10995         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
10996
10997         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
10998         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
10999         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11000         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11001         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11002
11003         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11004         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11005         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11006         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11007         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11008         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11009
11010         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11011         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11012         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11013         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11014         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11015         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11016
11017         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11018         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11019         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11020
11021         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11022         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11023         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11024         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11025         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11026         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11027
11028         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11029         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11030         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11031         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11032         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11033         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11034         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11035         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11036         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11037
11038         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11039         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11040         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11041         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11042         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11043         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11044         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11045         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11046         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11047
11048         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11049         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11050         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11051
11052         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11053         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11054         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11055         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11056         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11057         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11058         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11059         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11060         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11061         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11062         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11063         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11064
11065         case INSTR_TABLE: return NULL;
11066         case INSTR_TABLE_AF: return NULL;
11067         case INSTR_SELECTOR: return NULL;
11068         case INSTR_LEARNER: return NULL;
11069         case INSTR_LEARNER_AF: return NULL;
11070
11071         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11072         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11073
11074         case INSTR_EXTERN_OBJ: return NULL;
11075         case INSTR_EXTERN_FUNC: return NULL;
11076
11077         case INSTR_JMP: return NULL;
11078         case INSTR_JMP_VALID: return NULL;
11079         case INSTR_JMP_INVALID: return NULL;
11080         case INSTR_JMP_HIT: return NULL;
11081         case INSTR_JMP_MISS: return NULL;
11082         case INSTR_JMP_ACTION_HIT: return NULL;
11083         case INSTR_JMP_ACTION_MISS: return NULL;
11084         case INSTR_JMP_EQ: return NULL;
11085         case INSTR_JMP_EQ_MH: return NULL;
11086         case INSTR_JMP_EQ_HM: return NULL;
11087         case INSTR_JMP_EQ_HH: return NULL;
11088         case INSTR_JMP_EQ_I: return NULL;
11089         case INSTR_JMP_NEQ: return NULL;
11090         case INSTR_JMP_NEQ_MH: return NULL;
11091         case INSTR_JMP_NEQ_HM: return NULL;
11092         case INSTR_JMP_NEQ_HH: return NULL;
11093         case INSTR_JMP_NEQ_I: return NULL;
11094         case INSTR_JMP_LT: return NULL;
11095         case INSTR_JMP_LT_MH: return NULL;
11096         case INSTR_JMP_LT_HM: return NULL;
11097         case INSTR_JMP_LT_HH: return NULL;
11098         case INSTR_JMP_LT_MI: return NULL;
11099         case INSTR_JMP_LT_HI: return NULL;
11100         case INSTR_JMP_GT: return NULL;
11101         case INSTR_JMP_GT_MH: return NULL;
11102         case INSTR_JMP_GT_HM: return NULL;
11103         case INSTR_JMP_GT_HH: return NULL;
11104         case INSTR_JMP_GT_MI: return NULL;
11105         case INSTR_JMP_GT_HI: return NULL;
11106
11107         case INSTR_RETURN: return NULL;
11108
11109         default: return NULL;
11110         }
11111 }
11112
11113 static void
11114 action_instr_does_tx_codegen(struct action *a,
11115                         uint32_t instr_pos,
11116                         struct instruction *instr,
11117                         FILE *f)
11118 {
11119         fprintf(f,
11120                 "%s(p, t, &action_%s_instructions[%u]);\n"
11121                 "\tthread_ip_reset(p, t);\n"
11122                 "\tinstr_rx_exec(p);\n"
11123                 "\treturn;\n",
11124                 instr_type_to_func(instr),
11125                 a->name,
11126                 instr_pos);
11127 }
11128
11129 static void
11130 action_instr_extern_obj_codegen(struct action *a,
11131                                 uint32_t instr_pos,
11132                                 FILE *f)
11133 {
11134         fprintf(f,
11135                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11136                 a->name,
11137                 instr_pos);
11138 }
11139
11140 static void
11141 action_instr_extern_func_codegen(struct action *a,
11142                                  uint32_t instr_pos,
11143                                  FILE *f)
11144 {
11145         fprintf(f,
11146                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11147                 a->name,
11148                 instr_pos);
11149 }
11150
11151 static void
11152 action_instr_jmp_codegen(struct action *a,
11153                          uint32_t instr_pos,
11154                          struct instruction *instr,
11155                          struct instruction_data *data,
11156                          FILE *f)
11157 {
11158         switch (instr->type) {
11159         case INSTR_JMP:
11160                 fprintf(f,
11161                         "goto %s;\n",
11162                         data->jmp_label);
11163                 return;
11164
11165         case INSTR_JMP_VALID:
11166                 fprintf(f,
11167                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11168                         "\t\tgoto %s;\n",
11169                         a->name,
11170                         instr_pos,
11171                         data->jmp_label);
11172                 return;
11173
11174         case INSTR_JMP_INVALID:
11175                 fprintf(f,
11176                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11177                         "\t\tgoto %s;\n",
11178                         a->name,
11179                         instr_pos,
11180                         data->jmp_label);
11181                 return;
11182
11183         case INSTR_JMP_HIT:
11184                 fprintf(f,
11185                         "if (t->hit)\n"
11186                         "\t\tgoto %s;\n",
11187                         data->jmp_label);
11188                 return;
11189
11190         case INSTR_JMP_MISS:
11191                 fprintf(f,
11192                         "if (!t->hit)\n"
11193                         "\t\tgoto %s;\n",
11194                         data->jmp_label);
11195                 return;
11196
11197         case INSTR_JMP_ACTION_HIT:
11198                 fprintf(f,
11199                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11200                         "\t\tgoto %s;\n",
11201                         a->name,
11202                         instr_pos,
11203                         data->jmp_label);
11204                 return;
11205
11206         case INSTR_JMP_ACTION_MISS:
11207                 fprintf(f,
11208                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11209                         "\t\tgoto %s;\n",
11210                         a->name,
11211                         instr_pos,
11212                         data->jmp_label);
11213                 return;
11214
11215         case INSTR_JMP_EQ:
11216                 fprintf(f,
11217                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11218                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11219                         "\t\tgoto %s;\n",
11220                         a->name,
11221                         instr_pos,
11222                         a->name,
11223                         instr_pos,
11224                         data->jmp_label);
11225                 return;
11226
11227         case INSTR_JMP_EQ_MH:
11228                 fprintf(f,
11229                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11230                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11231                         "\t\tgoto %s;\n",
11232                         a->name,
11233                         instr_pos,
11234                         a->name,
11235                         instr_pos,
11236                         data->jmp_label);
11237                 return;
11238
11239         case INSTR_JMP_EQ_HM:
11240                 fprintf(f,
11241                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11242                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11243                         "\t\tgoto %s;\n",
11244                         a->name,
11245                         instr_pos,
11246                         a->name,
11247                         instr_pos,
11248                         data->jmp_label);
11249                 return;
11250
11251         case INSTR_JMP_EQ_HH:
11252                 fprintf(f,
11253                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11254                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11255                         "\t\tgoto %s;\n",
11256                         a->name,
11257                         instr_pos,
11258                         a->name,
11259                         instr_pos,
11260                         data->jmp_label);
11261                 return;
11262
11263         case INSTR_JMP_EQ_I:
11264                 fprintf(f,
11265                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11266                         "action_%s_instructions[%u].jmp.b_val)\n"
11267                         "\t\tgoto %s;\n",
11268                         a->name,
11269                         instr_pos,
11270                         a->name,
11271                         instr_pos,
11272                         data->jmp_label);
11273                 return;
11274
11275         case INSTR_JMP_NEQ:
11276                 fprintf(f,
11277                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11278                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11279                         "\t\tgoto %s;\n",
11280                         a->name,
11281                         instr_pos,
11282                         a->name,
11283                         instr_pos,
11284                         data->jmp_label);
11285                 return;
11286
11287         case INSTR_JMP_NEQ_MH:
11288                 fprintf(f,
11289                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11290                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11291                         "\t\tgoto %s;\n",
11292                         a->name,
11293                         instr_pos,
11294                         a->name,
11295                         instr_pos,
11296                         data->jmp_label);
11297                 return;
11298
11299         case INSTR_JMP_NEQ_HM:
11300                 fprintf(f,
11301                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11302                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11303                         "\t\tgoto %s;\n",
11304                         a->name,
11305                         instr_pos,
11306                         a->name,
11307                         instr_pos,
11308                         data->jmp_label);
11309                 return;
11310
11311         case INSTR_JMP_NEQ_HH:
11312                 fprintf(f,
11313                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11314                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11315                         "\t\tgoto %s;\n",
11316                         a->name,
11317                         instr_pos,
11318                         a->name,
11319                         instr_pos,
11320                         data->jmp_label);
11321                 return;
11322
11323         case INSTR_JMP_NEQ_I:
11324                 fprintf(f,
11325                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11326                         "action_%s_instructions[%u].jmp.b_val)\n"
11327                         "\t\tgoto %s;\n",
11328                         a->name,
11329                         instr_pos,
11330                         a->name,
11331                         instr_pos,
11332                         data->jmp_label);
11333                 return;
11334
11335         case INSTR_JMP_LT:
11336                 fprintf(f,
11337                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11338                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11339                         "\t\tgoto %s;\n",
11340                         a->name,
11341                         instr_pos,
11342                         a->name,
11343                         instr_pos,
11344                         data->jmp_label);
11345                 return;
11346
11347         case INSTR_JMP_LT_MH:
11348                 fprintf(f,
11349                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11350                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11351                         "\t\tgoto %s;\n",
11352                         a->name,
11353                         instr_pos,
11354                         a->name,
11355                         instr_pos,
11356                         data->jmp_label);
11357                 return;
11358
11359         case INSTR_JMP_LT_HM:
11360                 fprintf(f,
11361                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11362                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11363                         "\t\tgoto %s;\n",
11364                         a->name,
11365                         instr_pos,
11366                         a->name,
11367                         instr_pos,
11368                         data->jmp_label);
11369                 return;
11370
11371         case INSTR_JMP_LT_HH:
11372                 fprintf(f,
11373                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11374                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11375                         "\t\tgoto %s;\n",
11376                         a->name,
11377                         instr_pos,
11378                         a->name,
11379                         instr_pos,
11380                         data->jmp_label);
11381                 return;
11382
11383         case INSTR_JMP_LT_MI:
11384                 fprintf(f,
11385                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11386                         "action_%s_instructions[%u].jmp.b_val)\n"
11387                         "\t\tgoto %s;\n",
11388                         a->name,
11389                         instr_pos,
11390                         a->name,
11391                         instr_pos,
11392                         data->jmp_label);
11393                 return;
11394
11395         case INSTR_JMP_LT_HI:
11396                 fprintf(f,
11397                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11398                         "action_%s_instructions[%u].jmp.b_val)\n"
11399                         "\t\tgoto %s;\n",
11400                         a->name,
11401                         instr_pos,
11402                         a->name,
11403                         instr_pos,
11404                         data->jmp_label);
11405                 return;
11406
11407         case INSTR_JMP_GT:
11408                 fprintf(f,
11409                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11410                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11411                         "\t\tgoto %s;\n",
11412                         a->name,
11413                         instr_pos,
11414                         a->name,
11415                         instr_pos,
11416                         data->jmp_label);
11417                 return;
11418
11419         case INSTR_JMP_GT_MH:
11420                 fprintf(f,
11421                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11422                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11423                         "\t\tgoto %s;\n",
11424                         a->name,
11425                         instr_pos,
11426                         a->name,
11427                         instr_pos,
11428                         data->jmp_label);
11429                 return;
11430
11431         case INSTR_JMP_GT_HM:
11432                 fprintf(f,
11433                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11434                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11435                         "\t\tgoto %s;\n",
11436                         a->name,
11437                         instr_pos,
11438                         a->name,
11439                         instr_pos,
11440                         data->jmp_label);
11441                 return;
11442
11443         case INSTR_JMP_GT_HH:
11444                 fprintf(f,
11445                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11446                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11447                         "\t\tgoto %s;\n",
11448                         a->name,
11449                         instr_pos,
11450                         a->name,
11451                         instr_pos,
11452                         data->jmp_label);
11453                 return;
11454
11455         case INSTR_JMP_GT_MI:
11456                 fprintf(f,
11457                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11458                         "action_%s_instructions[%u].jmp.b_val)\n"
11459                         "\t\tgoto %s;\n",
11460                         a->name,
11461                         instr_pos,
11462                         a->name,
11463                         instr_pos,
11464                         data->jmp_label);
11465                 return;
11466
11467         case INSTR_JMP_GT_HI:
11468                 fprintf(f,
11469                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11470                         "action_%s_instructions[%u].jmp.b_val)\n"
11471                         "\t\tgoto %s;\n",
11472                         a->name,
11473                         instr_pos,
11474                         a->name,
11475                         instr_pos,
11476                         data->jmp_label);
11477                 return;
11478
11479         default:
11480                 return;
11481         }
11482 }
11483
11484 static void
11485 action_instr_return_codegen(FILE *f)
11486 {
11487         fprintf(f,
11488                 "return;\n");
11489 }
11490
11491 static void
11492 action_instr_codegen(struct action *a, FILE *f)
11493 {
11494         uint32_t i;
11495
11496         fprintf(f,
11497                 "void\n"
11498                 "action_%s_run(struct rte_swx_pipeline *p)\n"
11499                 "{\n"
11500                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11501                 "\n",
11502                 a->name);
11503
11504         for (i = 0; i < a->n_instructions; i++) {
11505                 struct instruction *instr = &a->instructions[i];
11506                 struct instruction_data *data = &a->instruction_data[i];
11507
11508                 /* Label, if present. */
11509                 if (data->label[0])
11510                         fprintf(f, "\n%s : ", data->label);
11511                 else
11512                         fprintf(f, "\n\t");
11513
11514                 /* TX instruction type. */
11515                 if (instruction_does_tx(instr)) {
11516                         action_instr_does_tx_codegen(a, i, instr, f);
11517                         continue;
11518                 }
11519
11520                 /* Extern object/function instruction type. */
11521                 if (instr->type == INSTR_EXTERN_OBJ) {
11522                         action_instr_extern_obj_codegen(a, i, f);
11523                         continue;
11524                 }
11525
11526                 if (instr->type == INSTR_EXTERN_FUNC) {
11527                         action_instr_extern_func_codegen(a, i, f);
11528                         continue;
11529                 }
11530
11531                 /* Jump instruction type. */
11532                 if (instruction_is_jmp(instr)) {
11533                         action_instr_jmp_codegen(a, i, instr, data, f);
11534                         continue;
11535                 }
11536
11537                 /* Return instruction type. */
11538                 if (instr->type == INSTR_RETURN) {
11539                         action_instr_return_codegen(f);
11540                         continue;
11541                 }
11542
11543                 /* Any other instruction type. */
11544                 fprintf(f,
11545                         "%s(p, t, &action_%s_instructions[%u]);\n",
11546                         instr_type_to_func(instr),
11547                         a->name,
11548                         i);
11549         }
11550
11551         fprintf(f, "}\n\n");
11552 }
11553
11554 struct instruction_group {
11555         TAILQ_ENTRY(instruction_group) node;
11556
11557         uint32_t group_id;
11558
11559         uint32_t first_instr_id;
11560
11561         uint32_t last_instr_id;
11562
11563         instr_exec_t func;
11564 };
11565
11566 TAILQ_HEAD(instruction_group_list, instruction_group);
11567
11568 static struct instruction_group *
11569 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11570 {
11571         struct instruction_group *g;
11572
11573         TAILQ_FOREACH(g, igl, node)
11574                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11575                         return g;
11576
11577         return NULL;
11578 }
11579
11580 static void
11581 instruction_group_list_free(struct instruction_group_list *igl)
11582 {
11583         if (!igl)
11584                 return;
11585
11586         for ( ; ; ) {
11587                 struct instruction_group *g;
11588
11589                 g = TAILQ_FIRST(igl);
11590                 if (!g)
11591                         break;
11592
11593                 TAILQ_REMOVE(igl, g, node);
11594                 free(g);
11595         }
11596
11597         free(igl);
11598 }
11599
11600 static struct instruction_group_list *
11601 instruction_group_list_create(struct rte_swx_pipeline *p)
11602 {
11603         struct instruction_group_list *igl = NULL;
11604         struct instruction_group *g = NULL;
11605         uint32_t n_groups = 0, i;
11606
11607         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11608                 goto error;
11609
11610         /* List init. */
11611         igl = calloc(1, sizeof(struct instruction_group_list));
11612         if (!igl)
11613                 goto error;
11614
11615         TAILQ_INIT(igl);
11616
11617         /* Allocate the first group. */
11618         g = calloc(1, sizeof(struct instruction_group));
11619         if (!g)
11620                 goto error;
11621
11622         /* Iteration 1: Separate the instructions into groups based on the thread yield
11623          * instructions. Do not worry about the jump instructions at this point.
11624          */
11625         for (i = 0; i < p->n_instructions; i++) {
11626                 struct instruction *instr = &p->instructions[i];
11627
11628                 /* Check for thread yield instructions. */
11629                 if (!instruction_does_thread_yield(instr))
11630                         continue;
11631
11632                 /* If the current group contains at least one instruction, then finalize it (with
11633                  * the previous instruction), add it to the list and allocate a new group (that
11634                  * starts with the current instruction).
11635                  */
11636                 if (i - g->first_instr_id) {
11637                         /* Finalize the group. */
11638                         g->last_instr_id = i - 1;
11639
11640                         /* Add the group to the list. Advance the number of groups. */
11641                         TAILQ_INSERT_TAIL(igl, g, node);
11642                         n_groups++;
11643
11644                         /* Allocate a new group. */
11645                         g = calloc(1, sizeof(struct instruction_group));
11646                         if (!g)
11647                                 goto error;
11648
11649                         /* Initialize the new group. */
11650                         g->group_id = n_groups;
11651                         g->first_instr_id = i;
11652                 }
11653
11654                 /* Finalize the current group (with the current instruction, therefore this group
11655                  * contains just the current thread yield instruction), add it to the list and
11656                  * allocate a new group (that starts with the next instruction).
11657                  */
11658
11659                 /* Finalize the group. */
11660                 g->last_instr_id = i;
11661
11662                 /* Add the group to the list. Advance the number of groups. */
11663                 TAILQ_INSERT_TAIL(igl, g, node);
11664                 n_groups++;
11665
11666                 /* Allocate a new group. */
11667                 g = calloc(1, sizeof(struct instruction_group));
11668                 if (!g)
11669                         goto error;
11670
11671                 /* Initialize the new group. */
11672                 g->group_id = n_groups;
11673                 g->first_instr_id = i + 1;
11674         }
11675
11676         /* Handle the last group. */
11677         if (i - g->first_instr_id) {
11678                 /* Finalize the group. */
11679                 g->last_instr_id = i - 1;
11680
11681                 /* Add the group to the list. Advance the number of groups. */
11682                 TAILQ_INSERT_TAIL(igl, g, node);
11683                 n_groups++;
11684         } else
11685                 free(g);
11686
11687         g = NULL;
11688
11689         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11690          * the destination of a jump instruction located in a different group ("far jump"), then the
11691          * current group has to be split, so that the instruction representing the far jump
11692          * destination is at the start of its group.
11693          */
11694         for ( ; ; ) {
11695                 int is_modified = 0;
11696
11697                 for (i = 0; i < p->n_instructions; i++) {
11698                         struct instruction_data *data = &p->instruction_data[i];
11699                         struct instruction_group *g;
11700                         uint32_t j;
11701
11702                         /* Continue when the current instruction is not a jump destination. */
11703                         if (!data->n_users)
11704                                 continue;
11705
11706                         g = instruction_group_list_group_find(igl, i);
11707                         if (!g)
11708                                 goto error;
11709
11710                         /* Find out all the jump instructions with this destination. */
11711                         for (j = 0; j < p->n_instructions; j++) {
11712                                 struct instruction *jmp_instr = &p->instructions[j];
11713                                 struct instruction_data *jmp_data = &p->instruction_data[j];
11714                                 struct instruction_group *jmp_g, *new_g;
11715
11716                                 /* Continue when not a jump instruction. Even when jump instruction,
11717                                  * continue when the jump destination is not this instruction.
11718                                  */
11719                                 if (!instruction_is_jmp(jmp_instr) ||
11720                                     strcmp(jmp_data->jmp_label, data->label))
11721                                         continue;
11722
11723                                 jmp_g = instruction_group_list_group_find(igl, j);
11724                                 if (!jmp_g)
11725                                         goto error;
11726
11727                                 /* Continue when both the jump instruction and the jump destination
11728                                  * instruction are in the same group. Even when in different groups,
11729                                  * still continue if the jump destination instruction is already the
11730                                  * first instruction of its group.
11731                                  */
11732                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11733                                         continue;
11734
11735                                 /* Split the group of the current jump destination instruction to
11736                                  * make this instruction the first instruction of a new group.
11737                                  */
11738                                 new_g = calloc(1, sizeof(struct instruction_group));
11739                                 if (!new_g)
11740                                         goto error;
11741
11742                                 new_g->group_id = n_groups;
11743                                 new_g->first_instr_id = i;
11744                                 new_g->last_instr_id = g->last_instr_id;
11745
11746                                 g->last_instr_id = i - 1;
11747
11748                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11749                                 n_groups++;
11750                                 is_modified = 1;
11751
11752                                 /* The decision to split this group (to make the current instruction
11753                                  * the first instruction of a new group) is already taken and fully
11754                                  * implemented, so no need to search for more reasons to do it.
11755                                  */
11756                                 break;
11757                         }
11758                 }
11759
11760                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11761                  * previously considered local (i.e. the jump destination is in the same group as
11762                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11763                  * different group than the jump instruction). Wost case scenario: each instruction
11764                  * that is a jump destination ends up as the first instruction of its group.
11765                  */
11766                 if (!is_modified)
11767                         break;
11768         }
11769
11770         /* Re-assign the group IDs to be in incremental order. */
11771         i = 0;
11772         TAILQ_FOREACH(g, igl, node) {
11773                 g->group_id = i;
11774
11775                 i++;
11776         }
11777
11778         return igl;
11779
11780 error:
11781         instruction_group_list_free(igl);
11782
11783         free(g);
11784
11785         return NULL;
11786 }
11787
11788 static void
11789 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11790                                uint32_t instr_pos,
11791                                struct instruction *instr,
11792                                FILE *f)
11793 {
11794         fprintf(f,
11795                 "%s(p, t, &pipeline_instructions[%u]);\n"
11796                 "\tthread_ip_reset(p, t);\n"
11797                 "\tinstr_rx_exec(p);\n"
11798                 "\treturn;\n",
11799                 instr_type_to_func(instr),
11800                 instr_pos);
11801 }
11802
11803 static int
11804 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11805                            struct instruction_group_list *igl,
11806                            uint32_t jmp_instr_id,
11807                            struct instruction *jmp_instr,
11808                            struct instruction_data *jmp_data,
11809                            FILE *f)
11810 {
11811         struct instruction_group *jmp_g, *g;
11812         struct instruction_data *data;
11813         uint32_t instr_id;
11814
11815         switch (jmp_instr->type) {
11816         case INSTR_JMP:
11817                 break;
11818
11819         case INSTR_JMP_VALID:
11820                 fprintf(f,
11821                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11822                         jmp_instr_id);
11823                 break;
11824
11825         case INSTR_JMP_INVALID:
11826                 fprintf(f,
11827                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11828                         jmp_instr_id);
11829                 break;
11830
11831         case INSTR_JMP_HIT:
11832                 fprintf(f,
11833                         "if (t->hit)\n");
11834                 break;
11835
11836         case INSTR_JMP_MISS:
11837                 fprintf(f,
11838                         "if (!t->hit)\n");
11839                 break;
11840
11841         case INSTR_JMP_ACTION_HIT:
11842                 fprintf(f,
11843                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
11844                         jmp_instr_id);
11845                 break;
11846
11847         case INSTR_JMP_ACTION_MISS:
11848                 fprintf(f,
11849                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
11850                         jmp_instr_id);
11851                 break;
11852
11853         case INSTR_JMP_EQ:
11854                 fprintf(f,
11855                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11856                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11857                         jmp_instr_id,
11858                         jmp_instr_id);
11859                 break;
11860
11861         case INSTR_JMP_EQ_MH:
11862                 fprintf(f,
11863                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11864                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11865                         jmp_instr_id,
11866                         jmp_instr_id);
11867                 break;
11868
11869         case INSTR_JMP_EQ_HM:
11870                 fprintf(f,
11871                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11872                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11873                         jmp_instr_id,
11874                         jmp_instr_id);
11875                 break;
11876
11877         case INSTR_JMP_EQ_HH:
11878                 fprintf(f,
11879                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11880                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11881                         jmp_instr_id,
11882                         jmp_instr_id);
11883                 break;
11884
11885         case INSTR_JMP_EQ_I:
11886                 fprintf(f,
11887                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11888                         "pipeline_instructions[%u].jmp.b_val)",
11889                         jmp_instr_id,
11890                         jmp_instr_id);
11891                 break;
11892
11893         case INSTR_JMP_NEQ:
11894                 fprintf(f,
11895                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11896                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11897                         jmp_instr_id,
11898                         jmp_instr_id);
11899                 break;
11900
11901         case INSTR_JMP_NEQ_MH:
11902                 fprintf(f,
11903                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11904                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11905                         jmp_instr_id,
11906                         jmp_instr_id);
11907                 break;
11908
11909         case INSTR_JMP_NEQ_HM:
11910                 fprintf(f,
11911                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11912                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11913                         jmp_instr_id,
11914                         jmp_instr_id);
11915                 break;
11916
11917         case INSTR_JMP_NEQ_HH:
11918                 fprintf(f,
11919                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11920                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11921                         jmp_instr_id,
11922                         jmp_instr_id);
11923                 break;
11924
11925         case INSTR_JMP_NEQ_I:
11926                 fprintf(f,
11927                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11928                         "pipeline_instructions[%u].jmp.b_val)",
11929                         jmp_instr_id,
11930                         jmp_instr_id);
11931                 break;
11932
11933         case INSTR_JMP_LT:
11934                 fprintf(f,
11935                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11936                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11937                         jmp_instr_id,
11938                         jmp_instr_id);
11939                 break;
11940
11941         case INSTR_JMP_LT_MH:
11942                 fprintf(f,
11943                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11944                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11945                         jmp_instr_id,
11946                         jmp_instr_id);
11947                 break;
11948
11949         case INSTR_JMP_LT_HM:
11950                 fprintf(f,
11951                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11952                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11953                         jmp_instr_id,
11954                         jmp_instr_id);
11955                 break;
11956
11957         case INSTR_JMP_LT_HH:
11958                 fprintf(f,
11959                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11960                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11961                         jmp_instr_id,
11962                         jmp_instr_id);
11963                 break;
11964
11965         case INSTR_JMP_LT_MI:
11966                 fprintf(f,
11967                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11968                         "pipeline_instructions[%u].jmp.b_val)",
11969                         jmp_instr_id,
11970                         jmp_instr_id);
11971                 break;
11972
11973         case INSTR_JMP_LT_HI:
11974                 fprintf(f,
11975                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11976                         "pipeline_instructions[%u].jmp.b_val)",
11977                         jmp_instr_id,
11978                         jmp_instr_id);
11979                 break;
11980
11981         case INSTR_JMP_GT:
11982                 fprintf(f,
11983                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
11984                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11985                         jmp_instr_id,
11986                         jmp_instr_id);
11987                 break;
11988
11989         case INSTR_JMP_GT_MH:
11990                 fprintf(f,
11991                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
11992                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11993                         jmp_instr_id,
11994                         jmp_instr_id);
11995                 break;
11996
11997         case INSTR_JMP_GT_HM:
11998                 fprintf(f,
11999                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12000                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12001                         jmp_instr_id,
12002                         jmp_instr_id);
12003                 break;
12004
12005         case INSTR_JMP_GT_HH:
12006                 fprintf(f,
12007                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12008                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12009                         jmp_instr_id,
12010                         jmp_instr_id);
12011                 break;
12012
12013         case INSTR_JMP_GT_MI:
12014                 fprintf(f,
12015                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12016                         "pipeline_instructions[%u].jmp.b_val)",
12017                         jmp_instr_id,
12018                         jmp_instr_id);
12019                 break;
12020
12021         case INSTR_JMP_GT_HI:
12022                 fprintf(f,
12023                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12024                         "pipeline_instructions[%u].jmp.b_val)",
12025                         jmp_instr_id,
12026                         jmp_instr_id);
12027                 break;
12028
12029         default:
12030                 break;
12031         }
12032
12033         /* Find the instruction group of the jump instruction. */
12034         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12035         if (!jmp_g)
12036                 return -EINVAL;
12037
12038         /* Find the instruction group of the jump destination instruction. */
12039         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12040         if (!data)
12041                 return -EINVAL;
12042
12043         instr_id = data - p->instruction_data;
12044
12045         g = instruction_group_list_group_find(igl, instr_id);
12046         if (!g)
12047                 return -EINVAL;
12048
12049         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12050          * instruction group).
12051          */
12052         if (g->group_id == jmp_g->group_id)
12053                 fprintf(f,
12054                         "\n\t\tgoto %s;\n",
12055                         jmp_data->jmp_label);
12056         else
12057                 fprintf(f,
12058                         " {\n"
12059                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12060                         "\t\treturn;\n"
12061                         "\t}\n\n",
12062                         g->group_id);
12063
12064         return 0;
12065 }
12066
12067 static void
12068 instruction_group_list_codegen(struct instruction_group_list *igl,
12069                                struct rte_swx_pipeline *p,
12070                                FILE *f)
12071 {
12072         struct instruction_group *g;
12073         uint32_t i;
12074         int is_required = 0;
12075
12076         /* Check if code generation is required. */
12077         TAILQ_FOREACH(g, igl, node)
12078                 if (g->first_instr_id < g->last_instr_id)
12079                         is_required = 1;
12080
12081         if (!is_required)
12082                 return;
12083
12084         /* Generate the code for the pipeline instruction array. */
12085         fprintf(f,
12086                 "static const struct instruction pipeline_instructions[] = {\n");
12087
12088         for (i = 0; i < p->n_instructions; i++) {
12089                 struct instruction *instr = &p->instructions[i];
12090                 instruction_export_t func = export_table[instr->type];
12091
12092                 func(instr, f);
12093         }
12094
12095         fprintf(f, "};\n\n");
12096
12097         /* Generate the code for the pipeline functions: one function for each instruction group
12098          * that contains more than one instruction.
12099          */
12100         TAILQ_FOREACH(g, igl, node) {
12101                 struct instruction *last_instr;
12102                 uint32_t j;
12103
12104                 /* Skip if group contains a single instruction. */
12105                 if (g->last_instr_id == g->first_instr_id)
12106                         continue;
12107
12108                 /* Generate new pipeline function. */
12109                 fprintf(f,
12110                         "void\n"
12111                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12112                         "{\n"
12113                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12114                         "\n",
12115                         g->group_id);
12116
12117                 /* Generate the code for each pipeline instruction. */
12118                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12119                         struct instruction *instr = &p->instructions[j];
12120                         struct instruction_data *data = &p->instruction_data[j];
12121
12122                         /* Label, if present. */
12123                         if (data->label[0])
12124                                 fprintf(f, "\n%s : ", data->label);
12125                         else
12126                                 fprintf(f, "\n\t");
12127
12128                         /* TX instruction type. */
12129                         if (instruction_does_tx(instr)) {
12130                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12131                                 continue;
12132                         }
12133
12134                         /* Jump instruction type. */
12135                         if (instruction_is_jmp(instr)) {
12136                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12137                                 continue;
12138                         }
12139
12140                         /* Any other instruction type. */
12141                         fprintf(f,
12142                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12143                                 instr_type_to_func(instr),
12144                                 j);
12145                 }
12146
12147                 /* Finalize the generated pipeline function. For some instructions such as TX,
12148                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12149                  * decided unconditionally and the instruction pointer of the current thread set
12150                  * accordingly; for all the other instructions, the instruction pointer must be
12151                  * incremented now.
12152                  */
12153                 last_instr = &p->instructions[g->last_instr_id];
12154
12155                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12156                         fprintf(f,
12157                                 "thread_ip_inc(p);\n");
12158
12159                 fprintf(f,
12160                         "}\n"
12161                         "\n");
12162         }
12163 }
12164
12165 static uint32_t
12166 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12167 {
12168         struct instruction_group *g;
12169         uint32_t n_custom_instr = 0;
12170
12171         /* Groups with a single instruction: no function is generated for this group, the group
12172          * keeps its current instruction. Groups with more than two instructions: one function and
12173          * the associated custom instruction get generated for each such group.
12174          */
12175         TAILQ_FOREACH(g, igl, node) {
12176                 if (g->first_instr_id == g->last_instr_id)
12177                         continue;
12178
12179                 n_custom_instr++;
12180         }
12181
12182         return n_custom_instr;
12183 }
12184
12185 static int
12186 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12187 {
12188         struct action *a;
12189         FILE *f = NULL;
12190
12191         /* Create the .c file. */
12192         f = fopen("/tmp/pipeline.c", "w");
12193         if (!f)
12194                 return -EIO;
12195
12196         /* Include the .h file. */
12197         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12198
12199         /* Add the code for each action. */
12200         TAILQ_FOREACH(a, &p->actions, node) {
12201                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12202
12203                 action_data_codegen(a, f);
12204
12205                 fprintf(f, "\n");
12206
12207                 action_instr_codegen(a, f);
12208
12209                 fprintf(f, "\n");
12210         }
12211
12212         /* Add the pipeline code. */
12213         instruction_group_list_codegen(igl, p, f);
12214
12215         /* Close the .c file. */
12216         fclose(f);
12217
12218         return 0;
12219 }
12220
12221 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12222 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12223 #endif
12224
12225 static int
12226 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12227 {
12228         struct action *a;
12229         struct instruction_group *g;
12230         char *dir_in, *buffer = NULL;
12231         const char *dir_out;
12232         int status = 0;
12233
12234         /* Get the environment variables. */
12235         dir_in = getenv("RTE_INSTALL_DIR");
12236         if (!dir_in) {
12237                 status = -EINVAL;
12238                 goto free;
12239         }
12240
12241         dir_out = "/tmp";
12242
12243         /* Memory allocation for the command buffer. */
12244         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12245         if (!buffer) {
12246                 status = -ENOMEM;
12247                 goto free;
12248         }
12249
12250         snprintf(buffer,
12251                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12252                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12253                  "-I %s/lib/pipeline "
12254                  "-I %s/lib/eal/include "
12255                  "-I %s/lib/eal/x86/include "
12256                  "-I %s/lib/eal/include/generic "
12257                  "-I %s/lib/meter "
12258                  "-I %s/lib/port "
12259                  "-I %s/lib/table "
12260                  "-I %s/lib/pipeline "
12261                  "-I %s/config "
12262                  "-I %s/build "
12263                  "-I %s/lib/eal/linux/include "
12264                  ">%s/pipeline.log 2>&1 "
12265                  "&& "
12266                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12267                  ">>%s/pipeline.log 2>&1",
12268                  dir_out,
12269                  dir_out,
12270                  dir_in,
12271                  dir_in,
12272                  dir_in,
12273                  dir_in,
12274                  dir_in,
12275                  dir_in,
12276                  dir_in,
12277                  dir_in,
12278                  dir_in,
12279                  dir_in,
12280                  dir_in,
12281                  dir_out,
12282                  dir_out,
12283                  dir_out,
12284                  dir_out);
12285
12286         /* Build the shared object library. */
12287         status = system(buffer);
12288         if (status)
12289                 goto free;
12290
12291         /* Open library. */
12292         snprintf(buffer,
12293                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12294                  "%s/libpipeline.so",
12295                  dir_out);
12296
12297         p->lib = dlopen(buffer, RTLD_LAZY);
12298         if (!p->lib) {
12299                 status = -EIO;
12300                 goto free;
12301         }
12302
12303         /* Get the action function symbols. */
12304         TAILQ_FOREACH(a, &p->actions, node) {
12305                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12306
12307                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12308                 if (!p->action_funcs[a->id]) {
12309                         status = -EINVAL;
12310                         goto free;
12311                 }
12312         }
12313
12314         /* Get the pipeline function symbols. */
12315         TAILQ_FOREACH(g, igl, node) {
12316                 if (g->first_instr_id == g->last_instr_id)
12317                         continue;
12318
12319                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12320
12321                 g->func = dlsym(p->lib, buffer);
12322                 if (!g->func) {
12323                         status = -EINVAL;
12324                         goto free;
12325                 }
12326         }
12327
12328 free:
12329         if (status && p->lib) {
12330                 dlclose(p->lib);
12331                 p->lib = NULL;
12332         }
12333
12334         free(buffer);
12335
12336         return status;
12337 }
12338
12339 static int
12340 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12341                       struct instruction_group_list *igl)
12342 {
12343         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12344
12345         /* Check that enough space is available within the pipeline instruction table to store all
12346          * the custom instructions.
12347          */
12348         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12349                 return -ENOSPC;
12350
12351         return 0;
12352 }
12353
12354 static void
12355 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12356 {
12357         struct instruction_group *g;
12358         uint32_t i;
12359
12360         /* Pipeline table instructions. */
12361         for (i = 0; i < p->n_instructions; i++) {
12362                 struct instruction *instr = &p->instructions[i];
12363
12364                 if (instr->type == INSTR_TABLE)
12365                         instr->type = INSTR_TABLE_AF;
12366
12367                 if (instr->type == INSTR_LEARNER)
12368                         instr->type = INSTR_LEARNER_AF;
12369         }
12370
12371         /* Pipeline custom instructions. */
12372         i = 0;
12373         TAILQ_FOREACH(g, igl, node) {
12374                 struct instruction *instr = &p->instructions[g->first_instr_id];
12375                 uint32_t j;
12376
12377                 if (g->first_instr_id == g->last_instr_id)
12378                         continue;
12379
12380                 /* Install a new custom instruction. */
12381                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12382
12383                 /* First instruction of the group: change its type to the new custom instruction. */
12384                 instr->type = INSTR_CUSTOM_0 + i;
12385
12386                 /* All the subsequent instructions of the group: invalidate. */
12387                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12388                         struct instruction_data *data = &p->instruction_data[j];
12389
12390                         data->invalid = 1;
12391                 }
12392
12393                 i++;
12394         }
12395
12396         /* Remove the invalidated instructions. */
12397         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12398
12399         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12400          * instructions that are the only instruction within their group, so they were left
12401          * unmodified).
12402          */
12403         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12404 }
12405
12406 static int
12407 pipeline_compile(struct rte_swx_pipeline *p)
12408 {
12409         struct instruction_group_list *igl = NULL;
12410         int status = 0;
12411
12412         igl = instruction_group_list_create(p);
12413         if (!igl) {
12414                 status = -ENOMEM;
12415                 goto free;
12416         }
12417
12418         /* Code generation. */
12419         status = pipeline_codegen(p, igl);
12420         if (status)
12421                 goto free;
12422
12423         /* Build and load the shared object library. */
12424         status = pipeline_libload(p, igl);
12425         if (status)
12426                 goto free;
12427
12428         /* Adjust instructions. */
12429         status = pipeline_adjust_check(p, igl);
12430         if (status)
12431                 goto free;
12432
12433         pipeline_adjust(p, igl);
12434
12435 free:
12436         instruction_group_list_free(igl);
12437
12438         return status;
12439 }