lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8 #include <dlfcn.h>
   9
  10 #include "rte_swx_pipeline_internal.h"
  11
  12 #define CHECK(condition, err_code)                                             \
  13 do {                                                                           \
  14         if (!(condition))                                                      \
  15                 return -(err_code);                                            \
  16 } while (0)
  17
  18 #define CHECK_NAME(name, err_code)                                             \
  19         CHECK((name) &&                                                        \
  20               (name)[0] &&                                                     \
  21               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  22               err_code)
  23
  24 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  25         CHECK((instr) &&                                                       \
  26               (instr)[0] &&                                                    \
  27               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  28                RTE_SWX_INSTRUCTION_SIZE),                                      \
  29               err_code)
  30
  31 /*
  32  * Environment.
  33  */
  34 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  35
  36 #include <rte_malloc.h>
  37
  38 static void *
  39 env_malloc(size_t size, size_t alignment, int numa_node)
  40 {
  41         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  42 }
  43
  44 static void
  45 env_free(void *start, size_t size __rte_unused)
  46 {
  47         rte_free(start);
  48 }
  49
  50 #else
  51
  52 #include <numa.h>
  53
  54 static void *
  55 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  56 {
  57         void *start;
  58
  59         if (numa_available() == -1)
  60                 return NULL;
  61
  62         start = numa_alloc_onnode(size, numa_node);
  63         if (!start)
  64                 return NULL;
  65
  66         memset(start, 0, size);
  67         return start;
  68 }
  69
  70 static void
  71 env_free(void *start, size_t size)
  72 {
  73         if (numa_available() == -1)
  74                 return;
  75
  76         numa_free(start, size);
  77 }
  78
  79 #endif
  80
  81 /*
  82  * Struct.
  83  */
  84 static struct struct_type *
  85 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  86 {
  87         struct struct_type *elem;
  88
  89         TAILQ_FOREACH(elem, &p->struct_types, node)
  90                 if (strcmp(elem->name, name) == 0)
  91                         return elem;
  92
  93         return NULL;
  94 }
  95
  96 static struct field *
  97 struct_type_field_find(struct struct_type *st, const char *name)
  98 {
  99         uint32_t i;
 100
 101         for (i = 0; i < st->n_fields; i++) {
 102                 struct field *f = &st->fields[i];
 103
 104                 if (strcmp(f->name, name) == 0)
 105                         return f;
 106         }
 107
 108         return NULL;
 109 }
 110
 111 int
 112 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 113                                       const char *name,
 114                                       struct rte_swx_field_params *fields,
 115                                       uint32_t n_fields,
 116                                       int last_field_has_variable_size)
 117 {
 118         struct struct_type *st;
 119         uint32_t i;
 120
 121         CHECK(p, EINVAL);
 122         CHECK_NAME(name, EINVAL);
 123         CHECK(fields, EINVAL);
 124         CHECK(n_fields, EINVAL);
 125
 126         for (i = 0; i < n_fields; i++) {
 127                 struct rte_swx_field_params *f = &fields[i];
 128                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 129                 uint32_t j;
 130
 131                 CHECK_NAME(f->name, EINVAL);
 132                 CHECK(f->n_bits, EINVAL);
 133                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 134                 CHECK((f->n_bits & 7) == 0, EINVAL);
 135
 136                 for (j = 0; j < i; j++) {
 137                         struct rte_swx_field_params *f_prev = &fields[j];
 138
 139                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 140                 }
 141         }
 142
 143         CHECK(!struct_type_find(p, name), EEXIST);
 144
 145         /* Node allocation. */
 146         st = calloc(1, sizeof(struct struct_type));
 147         CHECK(st, ENOMEM);
 148
 149         st->fields = calloc(n_fields, sizeof(struct field));
 150         if (!st->fields) {
 151                 free(st);
 152                 CHECK(0, ENOMEM);
 153         }
 154
 155         /* Node initialization. */
 156         strcpy(st->name, name);
 157         for (i = 0; i < n_fields; i++) {
 158                 struct field *dst = &st->fields[i];
 159                 struct rte_swx_field_params *src = &fields[i];
 160                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 161
 162                 strcpy(dst->name, src->name);
 163                 dst->n_bits = src->n_bits;
 164                 dst->offset = st->n_bits;
 165                 dst->var_size = var_size;
 166
 167                 st->n_bits += src->n_bits;
 168                 st->n_bits_min += var_size ? 0 : src->n_bits;
 169         }
 170         st->n_fields = n_fields;
 171         st->var_size = last_field_has_variable_size;
 172
 173         /* Node add to tailq. */
 174         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 175
 176         return 0;
 177 }
 178
 179 static int
 180 struct_build(struct rte_swx_pipeline *p)
 181 {
 182         uint32_t i;
 183
 184         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 185                 struct thread *t = &p->threads[i];
 186
 187                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 188                 CHECK(t->structs, ENOMEM);
 189         }
 190
 191         return 0;
 192 }
 193
 194 static void
 195 struct_build_free(struct rte_swx_pipeline *p)
 196 {
 197         uint32_t i;
 198
 199         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 200                 struct thread *t = &p->threads[i];
 201
 202                 free(t->structs);
 203                 t->structs = NULL;
 204         }
 205 }
 206
 207 static void
 208 struct_free(struct rte_swx_pipeline *p)
 209 {
 210         struct_build_free(p);
 211
 212         /* Struct types. */
 213         for ( ; ; ) {
 214                 struct struct_type *elem;
 215
 216                 elem = TAILQ_FIRST(&p->struct_types);
 217                 if (!elem)
 218                         break;
 219
 220                 TAILQ_REMOVE(&p->struct_types, elem, node);
 221                 free(elem->fields);
 222                 free(elem);
 223         }
 224 }
 225
 226 /*
 227  * Input port.
 228  */
 229 static struct port_in_type *
 230 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 231 {
 232         struct port_in_type *elem;
 233
 234         if (!name)
 235                 return NULL;
 236
 237         TAILQ_FOREACH(elem, &p->port_in_types, node)
 238                 if (strcmp(elem->name, name) == 0)
 239                         return elem;
 240
 241         return NULL;
 242 }
 243
 244 int
 245 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 246                                        const char *name,
 247                                        struct rte_swx_port_in_ops *ops)
 248 {
 249         struct port_in_type *elem;
 250
 251         CHECK(p, EINVAL);
 252         CHECK_NAME(name, EINVAL);
 253         CHECK(ops, EINVAL);
 254         CHECK(ops->create, EINVAL);
 255         CHECK(ops->free, EINVAL);
 256         CHECK(ops->pkt_rx, EINVAL);
 257         CHECK(ops->stats_read, EINVAL);
 258
 259         CHECK(!port_in_type_find(p, name), EEXIST);
 260
 261         /* Node allocation. */
 262         elem = calloc(1, sizeof(struct port_in_type));
 263         CHECK(elem, ENOMEM);
 264
 265         /* Node initialization. */
 266         strcpy(elem->name, name);
 267         memcpy(&elem->ops, ops, sizeof(*ops));
 268
 269         /* Node add to tailq. */
 270         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 271
 272         return 0;
 273 }
 274
 275 static struct port_in *
 276 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 277 {
 278         struct port_in *port;
 279
 280         TAILQ_FOREACH(port, &p->ports_in, node)
 281                 if (port->id == port_id)
 282                         return port;
 283
 284         return NULL;
 285 }
 286
 287 int
 288 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 289                                 uint32_t port_id,
 290                                 const char *port_type_name,
 291                                 void *args)
 292 {
 293         struct port_in_type *type = NULL;
 294         struct port_in *port = NULL;
 295         void *obj = NULL;
 296
 297         CHECK(p, EINVAL);
 298
 299         CHECK(!port_in_find(p, port_id), EINVAL);
 300
 301         CHECK_NAME(port_type_name, EINVAL);
 302         type = port_in_type_find(p, port_type_name);
 303         CHECK(type, EINVAL);
 304
 305         obj = type->ops.create(args);
 306         CHECK(obj, ENODEV);
 307
 308         /* Node allocation. */
 309         port = calloc(1, sizeof(struct port_in));
 310         CHECK(port, ENOMEM);
 311
 312         /* Node initialization. */
 313         port->type = type;
 314         port->obj = obj;
 315         port->id = port_id;
 316
 317         /* Node add to tailq. */
 318         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 319         if (p->n_ports_in < port_id + 1)
 320                 p->n_ports_in = port_id + 1;
 321
 322         return 0;
 323 }
 324
 325 static int
 326 port_in_build(struct rte_swx_pipeline *p)
 327 {
 328         struct port_in *port;
 329         uint32_t i;
 330
 331         CHECK(p->n_ports_in, EINVAL);
 332         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 333
 334         for (i = 0; i < p->n_ports_in; i++)
 335                 CHECK(port_in_find(p, i), EINVAL);
 336
 337         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 338         CHECK(p->in, ENOMEM);
 339
 340         TAILQ_FOREACH(port, &p->ports_in, node) {
 341                 struct port_in_runtime *in = &p->in[port->id];
 342
 343                 in->pkt_rx = port->type->ops.pkt_rx;
 344                 in->obj = port->obj;
 345         }
 346
 347         return 0;
 348 }
 349
 350 static void
 351 port_in_build_free(struct rte_swx_pipeline *p)
 352 {
 353         free(p->in);
 354         p->in = NULL;
 355 }
 356
 357 static void
 358 port_in_free(struct rte_swx_pipeline *p)
 359 {
 360         port_in_build_free(p);
 361
 362         /* Input ports. */
 363         for ( ; ; ) {
 364                 struct port_in *port;
 365
 366                 port = TAILQ_FIRST(&p->ports_in);
 367                 if (!port)
 368                         break;
 369
 370                 TAILQ_REMOVE(&p->ports_in, port, node);
 371                 port->type->ops.free(port->obj);
 372                 free(port);
 373         }
 374
 375         /* Input port types. */
 376         for ( ; ; ) {
 377                 struct port_in_type *elem;
 378
 379                 elem = TAILQ_FIRST(&p->port_in_types);
 380                 if (!elem)
 381                         break;
 382
 383                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 384                 free(elem);
 385         }
 386 }
 387
 388 /*
 389  * Output port.
 390  */
 391 static struct port_out_type *
 392 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 393 {
 394         struct port_out_type *elem;
 395
 396         if (!name)
 397                 return NULL;
 398
 399         TAILQ_FOREACH(elem, &p->port_out_types, node)
 400                 if (!strcmp(elem->name, name))
 401                         return elem;
 402
 403         return NULL;
 404 }
 405
 406 int
 407 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 408                                         const char *name,
 409                                         struct rte_swx_port_out_ops *ops)
 410 {
 411         struct port_out_type *elem;
 412
 413         CHECK(p, EINVAL);
 414         CHECK_NAME(name, EINVAL);
 415         CHECK(ops, EINVAL);
 416         CHECK(ops->create, EINVAL);
 417         CHECK(ops->free, EINVAL);
 418         CHECK(ops->pkt_tx, EINVAL);
 419         CHECK(ops->stats_read, EINVAL);
 420
 421         CHECK(!port_out_type_find(p, name), EEXIST);
 422
 423         /* Node allocation. */
 424         elem = calloc(1, sizeof(struct port_out_type));
 425         CHECK(elem, ENOMEM);
 426
 427         /* Node initialization. */
 428         strcpy(elem->name, name);
 429         memcpy(&elem->ops, ops, sizeof(*ops));
 430
 431         /* Node add to tailq. */
 432         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 433
 434         return 0;
 435 }
 436
 437 static struct port_out *
 438 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 439 {
 440         struct port_out *port;
 441
 442         TAILQ_FOREACH(port, &p->ports_out, node)
 443                 if (port->id == port_id)
 444                         return port;
 445
 446         return NULL;
 447 }
 448
 449 int
 450 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 451                                  uint32_t port_id,
 452                                  const char *port_type_name,
 453                                  void *args)
 454 {
 455         struct port_out_type *type = NULL;
 456         struct port_out *port = NULL;
 457         void *obj = NULL;
 458
 459         CHECK(p, EINVAL);
 460
 461         CHECK(!port_out_find(p, port_id), EINVAL);
 462
 463         CHECK_NAME(port_type_name, EINVAL);
 464         type = port_out_type_find(p, port_type_name);
 465         CHECK(type, EINVAL);
 466
 467         obj = type->ops.create(args);
 468         CHECK(obj, ENODEV);
 469
 470         /* Node allocation. */
 471         port = calloc(1, sizeof(struct port_out));
 472         CHECK(port, ENOMEM);
 473
 474         /* Node initialization. */
 475         port->type = type;
 476         port->obj = obj;
 477         port->id = port_id;
 478
 479         /* Node add to tailq. */
 480         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 481         if (p->n_ports_out < port_id + 1)
 482                 p->n_ports_out = port_id + 1;
 483
 484         return 0;
 485 }
 486
 487 static int
 488 port_out_build(struct rte_swx_pipeline *p)
 489 {
 490         struct port_out *port;
 491         uint32_t i;
 492
 493         CHECK(p->n_ports_out, EINVAL);
 494
 495         for (i = 0; i < p->n_ports_out; i++)
 496                 CHECK(port_out_find(p, i), EINVAL);
 497
 498         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 499         CHECK(p->out, ENOMEM);
 500
 501         TAILQ_FOREACH(port, &p->ports_out, node) {
 502                 struct port_out_runtime *out = &p->out[port->id];
 503
 504                 out->pkt_tx = port->type->ops.pkt_tx;
 505                 out->flush = port->type->ops.flush;
 506                 out->obj = port->obj;
 507         }
 508
 509         return 0;
 510 }
 511
 512 static void
 513 port_out_build_free(struct rte_swx_pipeline *p)
 514 {
 515         free(p->out);
 516         p->out = NULL;
 517 }
 518
 519 static void
 520 port_out_free(struct rte_swx_pipeline *p)
 521 {
 522         port_out_build_free(p);
 523
 524         /* Output ports. */
 525         for ( ; ; ) {
 526                 struct port_out *port;
 527
 528                 port = TAILQ_FIRST(&p->ports_out);
 529                 if (!port)
 530                         break;
 531
 532                 TAILQ_REMOVE(&p->ports_out, port, node);
 533                 port->type->ops.free(port->obj);
 534                 free(port);
 535         }
 536
 537         /* Output port types. */
 538         for ( ; ; ) {
 539                 struct port_out_type *elem;
 540
 541                 elem = TAILQ_FIRST(&p->port_out_types);
 542                 if (!elem)
 543                         break;
 544
 545                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 546                 free(elem);
 547         }
 548 }
 549
 550 /*
 551  * Extern object.
 552  */
 553 static struct extern_type *
 554 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 555 {
 556         struct extern_type *elem;
 557
 558         TAILQ_FOREACH(elem, &p->extern_types, node)
 559                 if (strcmp(elem->name, name) == 0)
 560                         return elem;
 561
 562         return NULL;
 563 }
 564
 565 static struct extern_type_member_func *
 566 extern_type_member_func_find(struct extern_type *type, const char *name)
 567 {
 568         struct extern_type_member_func *elem;
 569
 570         TAILQ_FOREACH(elem, &type->funcs, node)
 571                 if (strcmp(elem->name, name) == 0)
 572                         return elem;
 573
 574         return NULL;
 575 }
 576
 577 static struct extern_obj *
 578 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 579 {
 580         struct extern_obj *elem;
 581
 582         TAILQ_FOREACH(elem, &p->extern_objs, node)
 583                 if (strcmp(elem->name, name) == 0)
 584                         return elem;
 585
 586         return NULL;
 587 }
 588
 589 static struct extern_type_member_func *
 590 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 591                              const char *name,
 592                              struct extern_obj **obj)
 593 {
 594         struct extern_obj *object;
 595         struct extern_type_member_func *func;
 596         char *object_name, *func_name;
 597
 598         if (name[0] != 'e' || name[1] != '.')
 599                 return NULL;
 600
 601         object_name = strdup(&name[2]);
 602         if (!object_name)
 603                 return NULL;
 604
 605         func_name = strchr(object_name, '.');
 606         if (!func_name) {
 607                 free(object_name);
 608                 return NULL;
 609         }
 610
 611         *func_name = 0;
 612         func_name++;
 613
 614         object = extern_obj_find(p, object_name);
 615         if (!object) {
 616                 free(object_name);
 617                 return NULL;
 618         }
 619
 620         func = extern_type_member_func_find(object->type, func_name);
 621         if (!func) {
 622                 free(object_name);
 623                 return NULL;
 624         }
 625
 626         if (obj)
 627                 *obj = object;
 628
 629         free(object_name);
 630         return func;
 631 }
 632
 633 static struct field *
 634 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 635                                const char *name,
 636                                struct extern_obj **object)
 637 {
 638         struct extern_obj *obj;
 639         struct field *f;
 640         char *obj_name, *field_name;
 641
 642         if ((name[0] != 'e') || (name[1] != '.'))
 643                 return NULL;
 644
 645         obj_name = strdup(&name[2]);
 646         if (!obj_name)
 647                 return NULL;
 648
 649         field_name = strchr(obj_name, '.');
 650         if (!field_name) {
 651                 free(obj_name);
 652                 return NULL;
 653         }
 654
 655         *field_name = 0;
 656         field_name++;
 657
 658         obj = extern_obj_find(p, obj_name);
 659         if (!obj) {
 660                 free(obj_name);
 661                 return NULL;
 662         }
 663
 664         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 665         if (!f) {
 666                 free(obj_name);
 667                 return NULL;
 668         }
 669
 670         if (object)
 671                 *object = obj;
 672
 673         free(obj_name);
 674         return f;
 675 }
 676
 677 int
 678 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 679         const char *name,
 680         const char *mailbox_struct_type_name,
 681         rte_swx_extern_type_constructor_t constructor,
 682         rte_swx_extern_type_destructor_t destructor)
 683 {
 684         struct extern_type *elem;
 685         struct struct_type *mailbox_struct_type;
 686
 687         CHECK(p, EINVAL);
 688
 689         CHECK_NAME(name, EINVAL);
 690         CHECK(!extern_type_find(p, name), EEXIST);
 691
 692         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 693         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 694         CHECK(mailbox_struct_type, EINVAL);
 695         CHECK(!mailbox_struct_type->var_size, EINVAL);
 696
 697         CHECK(constructor, EINVAL);
 698         CHECK(destructor, EINVAL);
 699
 700         /* Node allocation. */
 701         elem = calloc(1, sizeof(struct extern_type));
 702         CHECK(elem, ENOMEM);
 703
 704         /* Node initialization. */
 705         strcpy(elem->name, name);
 706         elem->mailbox_struct_type = mailbox_struct_type;
 707         elem->constructor = constructor;
 708         elem->destructor = destructor;
 709         TAILQ_INIT(&elem->funcs);
 710
 711         /* Node add to tailq. */
 712         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 713
 714         return 0;
 715 }
 716
 717 int
 718 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 719         const char *extern_type_name,
 720         const char *name,
 721         rte_swx_extern_type_member_func_t member_func)
 722 {
 723         struct extern_type *type;
 724         struct extern_type_member_func *type_member;
 725
 726         CHECK(p, EINVAL);
 727
 728         CHECK_NAME(extern_type_name, EINVAL);
 729         type = extern_type_find(p, extern_type_name);
 730         CHECK(type, EINVAL);
 731         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 732
 733         CHECK_NAME(name, EINVAL);
 734         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 735
 736         CHECK(member_func, EINVAL);
 737
 738         /* Node allocation. */
 739         type_member = calloc(1, sizeof(struct extern_type_member_func));
 740         CHECK(type_member, ENOMEM);
 741
 742         /* Node initialization. */
 743         strcpy(type_member->name, name);
 744         type_member->func = member_func;
 745         type_member->id = type->n_funcs;
 746
 747         /* Node add to tailq. */
 748         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 749         type->n_funcs++;
 750
 751         return 0;
 752 }
 753
 754 int
 755 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 756                                       const char *extern_type_name,
 757                                       const char *name,
 758                                       const char *args)
 759 {
 760         struct extern_type *type;
 761         struct extern_obj *obj;
 762         void *obj_handle;
 763
 764         CHECK(p, EINVAL);
 765
 766         CHECK_NAME(extern_type_name, EINVAL);
 767         type = extern_type_find(p, extern_type_name);
 768         CHECK(type, EINVAL);
 769
 770         CHECK_NAME(name, EINVAL);
 771         CHECK(!extern_obj_find(p, name), EEXIST);
 772
 773         /* Node allocation. */
 774         obj = calloc(1, sizeof(struct extern_obj));
 775         CHECK(obj, ENOMEM);
 776
 777         /* Object construction. */
 778         obj_handle = type->constructor(args);
 779         if (!obj_handle) {
 780                 free(obj);
 781                 CHECK(0, ENODEV);
 782         }
 783
 784         /* Node initialization. */
 785         strcpy(obj->name, name);
 786         obj->type = type;
 787         obj->obj = obj_handle;
 788         obj->struct_id = p->n_structs;
 789         obj->id = p->n_extern_objs;
 790
 791         /* Node add to tailq. */
 792         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 793         p->n_extern_objs++;
 794         p->n_structs++;
 795
 796         return 0;
 797 }
 798
 799 static int
 800 extern_obj_build(struct rte_swx_pipeline *p)
 801 {
 802         uint32_t i;
 803
 804         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 805                 struct thread *t = &p->threads[i];
 806                 struct extern_obj *obj;
 807
 808                 t->extern_objs = calloc(p->n_extern_objs,
 809                                         sizeof(struct extern_obj_runtime));
 810                 CHECK(t->extern_objs, ENOMEM);
 811
 812                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 813                         struct extern_obj_runtime *r =
 814                                 &t->extern_objs[obj->id];
 815                         struct extern_type_member_func *func;
 816                         uint32_t mailbox_size =
 817                                 obj->type->mailbox_struct_type->n_bits / 8;
 818
 819                         r->obj = obj->obj;
 820
 821                         r->mailbox = calloc(1, mailbox_size);
 822                         CHECK(r->mailbox, ENOMEM);
 823
 824                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 825                                 r->funcs[func->id] = func->func;
 826
 827                         t->structs[obj->struct_id] = r->mailbox;
 828                 }
 829         }
 830
 831         return 0;
 832 }
 833
 834 static void
 835 extern_obj_build_free(struct rte_swx_pipeline *p)
 836 {
 837         uint32_t i;
 838
 839         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 840                 struct thread *t = &p->threads[i];
 841                 uint32_t j;
 842
 843                 if (!t->extern_objs)
 844                         continue;
 845
 846                 for (j = 0; j < p->n_extern_objs; j++) {
 847                         struct extern_obj_runtime *r = &t->extern_objs[j];
 848
 849                         free(r->mailbox);
 850                 }
 851
 852                 free(t->extern_objs);
 853                 t->extern_objs = NULL;
 854         }
 855 }
 856
 857 static void
 858 extern_obj_free(struct rte_swx_pipeline *p)
 859 {
 860         extern_obj_build_free(p);
 861
 862         /* Extern objects. */
 863         for ( ; ; ) {
 864                 struct extern_obj *elem;
 865
 866                 elem = TAILQ_FIRST(&p->extern_objs);
 867                 if (!elem)
 868                         break;
 869
 870                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 871                 if (elem->obj)
 872                         elem->type->destructor(elem->obj);
 873                 free(elem);
 874         }
 875
 876         /* Extern types. */
 877         for ( ; ; ) {
 878                 struct extern_type *elem;
 879
 880                 elem = TAILQ_FIRST(&p->extern_types);
 881                 if (!elem)
 882                         break;
 883
 884                 TAILQ_REMOVE(&p->extern_types, elem, node);
 885
 886                 for ( ; ; ) {
 887                         struct extern_type_member_func *func;
 888
 889                         func = TAILQ_FIRST(&elem->funcs);
 890                         if (!func)
 891                                 break;
 892
 893                         TAILQ_REMOVE(&elem->funcs, func, node);
 894                         free(func);
 895                 }
 896
 897                 free(elem);
 898         }
 899 }
 900
 901 /*
 902  * Extern function.
 903  */
 904 static struct extern_func *
 905 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 906 {
 907         struct extern_func *elem;
 908
 909         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 910                 if (strcmp(elem->name, name) == 0)
 911                         return elem;
 912
 913         return NULL;
 914 }
 915
 916 static struct extern_func *
 917 extern_func_parse(struct rte_swx_pipeline *p,
 918                   const char *name)
 919 {
 920         if (name[0] != 'f' || name[1] != '.')
 921                 return NULL;
 922
 923         return extern_func_find(p, &name[2]);
 924 }
 925
 926 static struct field *
 927 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 928                                 const char *name,
 929                                 struct extern_func **function)
 930 {
 931         struct extern_func *func;
 932         struct field *f;
 933         char *func_name, *field_name;
 934
 935         if ((name[0] != 'f') || (name[1] != '.'))
 936                 return NULL;
 937
 938         func_name = strdup(&name[2]);
 939         if (!func_name)
 940                 return NULL;
 941
 942         field_name = strchr(func_name, '.');
 943         if (!field_name) {
 944                 free(func_name);
 945                 return NULL;
 946         }
 947
 948         *field_name = 0;
 949         field_name++;
 950
 951         func = extern_func_find(p, func_name);
 952         if (!func) {
 953                 free(func_name);
 954                 return NULL;
 955         }
 956
 957         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 958         if (!f) {
 959                 free(func_name);
 960                 return NULL;
 961         }
 962
 963         if (function)
 964                 *function = func;
 965
 966         free(func_name);
 967         return f;
 968 }
 969
 970 int
 971 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 972                                       const char *name,
 973                                       const char *mailbox_struct_type_name,
 974                                       rte_swx_extern_func_t func)
 975 {
 976         struct extern_func *f;
 977         struct struct_type *mailbox_struct_type;
 978
 979         CHECK(p, EINVAL);
 980
 981         CHECK_NAME(name, EINVAL);
 982         CHECK(!extern_func_find(p, name), EEXIST);
 983
 984         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 985         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 986         CHECK(mailbox_struct_type, EINVAL);
 987         CHECK(!mailbox_struct_type->var_size, EINVAL);
 988
 989         CHECK(func, EINVAL);
 990
 991         /* Node allocation. */
 992         f = calloc(1, sizeof(struct extern_func));
 993         CHECK(func, ENOMEM);
 994
 995         /* Node initialization. */
 996         strcpy(f->name, name);
 997         f->mailbox_struct_type = mailbox_struct_type;
 998         f->func = func;
 999         f->struct_id = p->n_structs;
1000         f->id = p->n_extern_funcs;
1001
1002         /* Node add to tailq. */
1003         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1004         p->n_extern_funcs++;
1005         p->n_structs++;
1006
1007         return 0;
1008 }
1009
1010 static int
1011 extern_func_build(struct rte_swx_pipeline *p)
1012 {
1013         uint32_t i;
1014
1015         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1016                 struct thread *t = &p->threads[i];
1017                 struct extern_func *func;
1018
1019                 /* Memory allocation. */
1020                 t->extern_funcs = calloc(p->n_extern_funcs,
1021                                          sizeof(struct extern_func_runtime));
1022                 CHECK(t->extern_funcs, ENOMEM);
1023
1024                 /* Extern function. */
1025                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1026                         struct extern_func_runtime *r =
1027                                 &t->extern_funcs[func->id];
1028                         uint32_t mailbox_size =
1029                                 func->mailbox_struct_type->n_bits / 8;
1030
1031                         r->func = func->func;
1032
1033                         r->mailbox = calloc(1, mailbox_size);
1034                         CHECK(r->mailbox, ENOMEM);
1035
1036                         t->structs[func->struct_id] = r->mailbox;
1037                 }
1038         }
1039
1040         return 0;
1041 }
1042
1043 static void
1044 extern_func_build_free(struct rte_swx_pipeline *p)
1045 {
1046         uint32_t i;
1047
1048         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1049                 struct thread *t = &p->threads[i];
1050                 uint32_t j;
1051
1052                 if (!t->extern_funcs)
1053                         continue;
1054
1055                 for (j = 0; j < p->n_extern_funcs; j++) {
1056                         struct extern_func_runtime *r = &t->extern_funcs[j];
1057
1058                         free(r->mailbox);
1059                 }
1060
1061                 free(t->extern_funcs);
1062                 t->extern_funcs = NULL;
1063         }
1064 }
1065
1066 static void
1067 extern_func_free(struct rte_swx_pipeline *p)
1068 {
1069         extern_func_build_free(p);
1070
1071         for ( ; ; ) {
1072                 struct extern_func *elem;
1073
1074                 elem = TAILQ_FIRST(&p->extern_funcs);
1075                 if (!elem)
1076                         break;
1077
1078                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1079                 free(elem);
1080         }
1081 }
1082
1083 /*
1084  * Header.
1085  */
1086 static struct header *
1087 header_find(struct rte_swx_pipeline *p, const char *name)
1088 {
1089         struct header *elem;
1090
1091         TAILQ_FOREACH(elem, &p->headers, node)
1092                 if (strcmp(elem->name, name) == 0)
1093                         return elem;
1094
1095         return NULL;
1096 }
1097
1098 static struct header *
1099 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1100 {
1101         struct header *elem;
1102
1103         TAILQ_FOREACH(elem, &p->headers, node)
1104                 if (elem->struct_id == struct_id)
1105                         return elem;
1106
1107         return NULL;
1108 }
1109
1110 static struct header *
1111 header_parse(struct rte_swx_pipeline *p,
1112              const char *name)
1113 {
1114         if (name[0] != 'h' || name[1] != '.')
1115                 return NULL;
1116
1117         return header_find(p, &name[2]);
1118 }
1119
1120 static struct field *
1121 header_field_parse(struct rte_swx_pipeline *p,
1122                    const char *name,
1123                    struct header **header)
1124 {
1125         struct header *h;
1126         struct field *f;
1127         char *header_name, *field_name;
1128
1129         if ((name[0] != 'h') || (name[1] != '.'))
1130                 return NULL;
1131
1132         header_name = strdup(&name[2]);
1133         if (!header_name)
1134                 return NULL;
1135
1136         field_name = strchr(header_name, '.');
1137         if (!field_name) {
1138                 free(header_name);
1139                 return NULL;
1140         }
1141
1142         *field_name = 0;
1143         field_name++;
1144
1145         h = header_find(p, header_name);
1146         if (!h) {
1147                 free(header_name);
1148                 return NULL;
1149         }
1150
1151         f = struct_type_field_find(h->st, field_name);
1152         if (!f) {
1153                 free(header_name);
1154                 return NULL;
1155         }
1156
1157         if (header)
1158                 *header = h;
1159
1160         free(header_name);
1161         return f;
1162 }
1163
1164 int
1165 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1166                                         const char *name,
1167                                         const char *struct_type_name)
1168 {
1169         struct struct_type *st;
1170         struct header *h;
1171         size_t n_headers_max;
1172
1173         CHECK(p, EINVAL);
1174         CHECK_NAME(name, EINVAL);
1175         CHECK_NAME(struct_type_name, EINVAL);
1176
1177         CHECK(!header_find(p, name), EEXIST);
1178
1179         st = struct_type_find(p, struct_type_name);
1180         CHECK(st, EINVAL);
1181
1182         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1183         CHECK(p->n_headers < n_headers_max, ENOSPC);
1184
1185         /* Node allocation. */
1186         h = calloc(1, sizeof(struct header));
1187         CHECK(h, ENOMEM);
1188
1189         /* Node initialization. */
1190         strcpy(h->name, name);
1191         h->st = st;
1192         h->struct_id = p->n_structs;
1193         h->id = p->n_headers;
1194
1195         /* Node add to tailq. */
1196         TAILQ_INSERT_TAIL(&p->headers, h, node);
1197         p->n_headers++;
1198         p->n_structs++;
1199
1200         return 0;
1201 }
1202
1203 static int
1204 header_build(struct rte_swx_pipeline *p)
1205 {
1206         struct header *h;
1207         uint32_t n_bytes = 0, i;
1208
1209         TAILQ_FOREACH(h, &p->headers, node) {
1210                 n_bytes += h->st->n_bits / 8;
1211         }
1212
1213         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1214                 struct thread *t = &p->threads[i];
1215                 uint32_t offset = 0;
1216
1217                 t->headers = calloc(p->n_headers,
1218                                     sizeof(struct header_runtime));
1219                 CHECK(t->headers, ENOMEM);
1220
1221                 t->headers_out = calloc(p->n_headers,
1222                                         sizeof(struct header_out_runtime));
1223                 CHECK(t->headers_out, ENOMEM);
1224
1225                 t->header_storage = calloc(1, n_bytes);
1226                 CHECK(t->header_storage, ENOMEM);
1227
1228                 t->header_out_storage = calloc(1, n_bytes);
1229                 CHECK(t->header_out_storage, ENOMEM);
1230
1231                 TAILQ_FOREACH(h, &p->headers, node) {
1232                         uint8_t *header_storage;
1233                         uint32_t n_bytes =  h->st->n_bits / 8;
1234
1235                         header_storage = &t->header_storage[offset];
1236                         offset += n_bytes;
1237
1238                         t->headers[h->id].ptr0 = header_storage;
1239                         t->headers[h->id].n_bytes = n_bytes;
1240
1241                         t->structs[h->struct_id] = header_storage;
1242                 }
1243         }
1244
1245         return 0;
1246 }
1247
1248 static void
1249 header_build_free(struct rte_swx_pipeline *p)
1250 {
1251         uint32_t i;
1252
1253         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1254                 struct thread *t = &p->threads[i];
1255
1256                 free(t->headers_out);
1257                 t->headers_out = NULL;
1258
1259                 free(t->headers);
1260                 t->headers = NULL;
1261
1262                 free(t->header_out_storage);
1263                 t->header_out_storage = NULL;
1264
1265                 free(t->header_storage);
1266                 t->header_storage = NULL;
1267         }
1268 }
1269
1270 static void
1271 header_free(struct rte_swx_pipeline *p)
1272 {
1273         header_build_free(p);
1274
1275         for ( ; ; ) {
1276                 struct header *elem;
1277
1278                 elem = TAILQ_FIRST(&p->headers);
1279                 if (!elem)
1280                         break;
1281
1282                 TAILQ_REMOVE(&p->headers, elem, node);
1283                 free(elem);
1284         }
1285 }
1286
1287 /*
1288  * Meta-data.
1289  */
1290 static struct field *
1291 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1292 {
1293         if (!p->metadata_st)
1294                 return NULL;
1295
1296         if (name[0] != 'm' || name[1] != '.')
1297                 return NULL;
1298
1299         return struct_type_field_find(p->metadata_st, &name[2]);
1300 }
1301
1302 int
1303 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1304                                           const char *struct_type_name)
1305 {
1306         struct struct_type *st = NULL;
1307
1308         CHECK(p, EINVAL);
1309
1310         CHECK_NAME(struct_type_name, EINVAL);
1311         st  = struct_type_find(p, struct_type_name);
1312         CHECK(st, EINVAL);
1313         CHECK(!st->var_size, EINVAL);
1314         CHECK(!p->metadata_st, EINVAL);
1315
1316         p->metadata_st = st;
1317         p->metadata_struct_id = p->n_structs;
1318
1319         p->n_structs++;
1320
1321         return 0;
1322 }
1323
1324 static int
1325 metadata_build(struct rte_swx_pipeline *p)
1326 {
1327         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1328         uint32_t i;
1329
1330         /* Thread-level initialization. */
1331         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1332                 struct thread *t = &p->threads[i];
1333                 uint8_t *metadata;
1334
1335                 metadata = calloc(1, n_bytes);
1336                 CHECK(metadata, ENOMEM);
1337
1338                 t->metadata = metadata;
1339                 t->structs[p->metadata_struct_id] = metadata;
1340         }
1341
1342         return 0;
1343 }
1344
1345 static void
1346 metadata_build_free(struct rte_swx_pipeline *p)
1347 {
1348         uint32_t i;
1349
1350         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1351                 struct thread *t = &p->threads[i];
1352
1353                 free(t->metadata);
1354                 t->metadata = NULL;
1355         }
1356 }
1357
1358 static void
1359 metadata_free(struct rte_swx_pipeline *p)
1360 {
1361         metadata_build_free(p);
1362 }
1363
1364 /*
1365  * Instruction.
1366  */
1367 static int
1368 instruction_is_tx(enum instruction_type type)
1369 {
1370         switch (type) {
1371         case INSTR_TX:
1372         case INSTR_TX_I:
1373                 return 1;
1374
1375         default:
1376                 return 0;
1377         }
1378 }
1379
1380 static int
1381 instruction_does_tx(struct instruction *instr)
1382 {
1383         switch (instr->type) {
1384         case INSTR_TX:
1385         case INSTR_TX_I:
1386         case INSTR_HDR_EMIT_TX:
1387         case INSTR_HDR_EMIT2_TX:
1388         case INSTR_HDR_EMIT3_TX:
1389         case INSTR_HDR_EMIT4_TX:
1390         case INSTR_HDR_EMIT5_TX:
1391         case INSTR_HDR_EMIT6_TX:
1392         case INSTR_HDR_EMIT7_TX:
1393         case INSTR_HDR_EMIT8_TX:
1394                 return 1;
1395         default:
1396                 return 0;
1397         }
1398 }
1399
1400 static int
1401 instruction_is_jmp(struct instruction *instr)
1402 {
1403         switch (instr->type) {
1404         case INSTR_JMP:
1405         case INSTR_JMP_VALID:
1406         case INSTR_JMP_INVALID:
1407         case INSTR_JMP_HIT:
1408         case INSTR_JMP_MISS:
1409         case INSTR_JMP_ACTION_HIT:
1410         case INSTR_JMP_ACTION_MISS:
1411         case INSTR_JMP_EQ:
1412         case INSTR_JMP_EQ_MH:
1413         case INSTR_JMP_EQ_HM:
1414         case INSTR_JMP_EQ_HH:
1415         case INSTR_JMP_EQ_I:
1416         case INSTR_JMP_NEQ:
1417         case INSTR_JMP_NEQ_MH:
1418         case INSTR_JMP_NEQ_HM:
1419         case INSTR_JMP_NEQ_HH:
1420         case INSTR_JMP_NEQ_I:
1421         case INSTR_JMP_LT:
1422         case INSTR_JMP_LT_MH:
1423         case INSTR_JMP_LT_HM:
1424         case INSTR_JMP_LT_HH:
1425         case INSTR_JMP_LT_MI:
1426         case INSTR_JMP_LT_HI:
1427         case INSTR_JMP_GT:
1428         case INSTR_JMP_GT_MH:
1429         case INSTR_JMP_GT_HM:
1430         case INSTR_JMP_GT_HH:
1431         case INSTR_JMP_GT_MI:
1432         case INSTR_JMP_GT_HI:
1433                 return 1;
1434
1435         default:
1436                 return 0;
1437         }
1438 }
1439
1440 static int
1441 instruction_does_thread_yield(struct instruction *instr)
1442 {
1443         switch (instr->type) {
1444         case INSTR_RX:
1445         case INSTR_TABLE:
1446         case INSTR_TABLE_AF:
1447         case INSTR_SELECTOR:
1448         case INSTR_LEARNER:
1449         case INSTR_LEARNER_AF:
1450         case INSTR_EXTERN_OBJ:
1451         case INSTR_EXTERN_FUNC:
1452                 return 1;
1453         default:
1454                 return 0;
1455         }
1456 }
1457
1458 static struct field *
1459 action_field_parse(struct action *action, const char *name);
1460
1461 static struct field *
1462 struct_field_parse(struct rte_swx_pipeline *p,
1463                    struct action *action,
1464                    const char *name,
1465                    uint32_t *struct_id)
1466 {
1467         struct field *f;
1468
1469         switch (name[0]) {
1470         case 'h':
1471         {
1472                 struct header *header;
1473
1474                 f = header_field_parse(p, name, &header);
1475                 if (!f)
1476                         return NULL;
1477
1478                 *struct_id = header->struct_id;
1479                 return f;
1480         }
1481
1482         case 'm':
1483         {
1484                 f = metadata_field_parse(p, name);
1485                 if (!f)
1486                         return NULL;
1487
1488                 *struct_id = p->metadata_struct_id;
1489                 return f;
1490         }
1491
1492         case 't':
1493         {
1494                 if (!action)
1495                         return NULL;
1496
1497                 f = action_field_parse(action, name);
1498                 if (!f)
1499                         return NULL;
1500
1501                 *struct_id = 0;
1502                 return f;
1503         }
1504
1505         case 'e':
1506         {
1507                 struct extern_obj *obj;
1508
1509                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1510                 if (!f)
1511                         return NULL;
1512
1513                 *struct_id = obj->struct_id;
1514                 return f;
1515         }
1516
1517         case 'f':
1518         {
1519                 struct extern_func *func;
1520
1521                 f = extern_func_mailbox_field_parse(p, name, &func);
1522                 if (!f)
1523                         return NULL;
1524
1525                 *struct_id = func->struct_id;
1526                 return f;
1527         }
1528
1529         default:
1530                 return NULL;
1531         }
1532 }
1533
1534 /*
1535  * rx.
1536  */
1537 static int
1538 instr_rx_translate(struct rte_swx_pipeline *p,
1539                    struct action *action,
1540                    char **tokens,
1541                    int n_tokens,
1542                    struct instruction *instr,
1543                    struct instruction_data *data __rte_unused)
1544 {
1545         struct field *f;
1546
1547         CHECK(!action, EINVAL);
1548         CHECK(n_tokens == 2, EINVAL);
1549
1550         f = metadata_field_parse(p, tokens[1]);
1551         CHECK(f, EINVAL);
1552
1553         instr->type = INSTR_RX;
1554         instr->io.io.offset = f->offset / 8;
1555         instr->io.io.n_bits = f->n_bits;
1556         return 0;
1557 }
1558
1559 /*
1560  * tx.
1561  */
1562 static int
1563 instr_tx_translate(struct rte_swx_pipeline *p,
1564                    struct action *action __rte_unused,
1565                    char **tokens,
1566                    int n_tokens,
1567                    struct instruction *instr,
1568                    struct instruction_data *data __rte_unused)
1569 {
1570         char *port = tokens[1];
1571         struct field *f;
1572         uint32_t port_val;
1573
1574         CHECK(n_tokens == 2, EINVAL);
1575
1576         f = metadata_field_parse(p, port);
1577         if (f) {
1578                 instr->type = INSTR_TX;
1579                 instr->io.io.offset = f->offset / 8;
1580                 instr->io.io.n_bits = f->n_bits;
1581                 return 0;
1582         }
1583
1584         /* TX_I. */
1585         port_val = strtoul(port, &port, 0);
1586         CHECK(!port[0], EINVAL);
1587
1588         instr->type = INSTR_TX_I;
1589         instr->io.io.val = port_val;
1590         return 0;
1591 }
1592
1593 static int
1594 instr_drop_translate(struct rte_swx_pipeline *p,
1595                      struct action *action __rte_unused,
1596                      char **tokens __rte_unused,
1597                      int n_tokens,
1598                      struct instruction *instr,
1599                      struct instruction_data *data __rte_unused)
1600 {
1601         CHECK(n_tokens == 1, EINVAL);
1602
1603         /* TX_I. */
1604         instr->type = INSTR_TX_I;
1605         instr->io.io.val = p->n_ports_out - 1;
1606         return 0;
1607 }
1608
1609 static inline void
1610 instr_tx_exec(struct rte_swx_pipeline *p)
1611 {
1612         struct thread *t = &p->threads[p->thread_id];
1613         struct instruction *ip = t->ip;
1614
1615         __instr_tx_exec(p, t, ip);
1616
1617         /* Thread. */
1618         thread_ip_reset(p, t);
1619         instr_rx_exec(p);
1620 }
1621
1622 static inline void
1623 instr_tx_i_exec(struct rte_swx_pipeline *p)
1624 {
1625         struct thread *t = &p->threads[p->thread_id];
1626         struct instruction *ip = t->ip;
1627
1628         __instr_tx_i_exec(p, t, ip);
1629
1630         /* Thread. */
1631         thread_ip_reset(p, t);
1632         instr_rx_exec(p);
1633 }
1634
1635 /*
1636  * extract.
1637  */
1638 static int
1639 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1640                             struct action *action,
1641                             char **tokens,
1642                             int n_tokens,
1643                             struct instruction *instr,
1644                             struct instruction_data *data __rte_unused)
1645 {
1646         struct header *h;
1647
1648         CHECK(!action, EINVAL);
1649         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1650
1651         h = header_parse(p, tokens[1]);
1652         CHECK(h, EINVAL);
1653
1654         if (n_tokens == 2) {
1655                 CHECK(!h->st->var_size, EINVAL);
1656
1657                 instr->type = INSTR_HDR_EXTRACT;
1658                 instr->io.hdr.header_id[0] = h->id;
1659                 instr->io.hdr.struct_id[0] = h->struct_id;
1660                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1661         } else {
1662                 struct field *mf;
1663
1664                 CHECK(h->st->var_size, EINVAL);
1665
1666                 mf = metadata_field_parse(p, tokens[2]);
1667                 CHECK(mf, EINVAL);
1668                 CHECK(!mf->var_size, EINVAL);
1669
1670                 instr->type = INSTR_HDR_EXTRACT_M;
1671                 instr->io.io.offset = mf->offset / 8;
1672                 instr->io.io.n_bits = mf->n_bits;
1673                 instr->io.hdr.header_id[0] = h->id;
1674                 instr->io.hdr.struct_id[0] = h->struct_id;
1675                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1676         }
1677
1678         return 0;
1679 }
1680
1681 static int
1682 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1683                               struct action *action,
1684                               char **tokens,
1685                               int n_tokens,
1686                               struct instruction *instr,
1687                               struct instruction_data *data __rte_unused)
1688 {
1689         struct header *h;
1690
1691         CHECK(!action, EINVAL);
1692         CHECK(n_tokens == 2, EINVAL);
1693
1694         h = header_parse(p, tokens[1]);
1695         CHECK(h, EINVAL);
1696         CHECK(!h->st->var_size, EINVAL);
1697
1698         instr->type = INSTR_HDR_LOOKAHEAD;
1699         instr->io.hdr.header_id[0] = h->id;
1700         instr->io.hdr.struct_id[0] = h->struct_id;
1701         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1702
1703         return 0;
1704 }
1705
1706 static inline void
1707 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1708 {
1709         struct thread *t = &p->threads[p->thread_id];
1710         struct instruction *ip = t->ip;
1711
1712         __instr_hdr_extract_exec(p, t, ip);
1713
1714         /* Thread. */
1715         thread_ip_inc(p);
1716 }
1717
1718 static inline void
1719 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1720 {
1721         struct thread *t = &p->threads[p->thread_id];
1722         struct instruction *ip = t->ip;
1723
1724         __instr_hdr_extract2_exec(p, t, ip);
1725
1726         /* Thread. */
1727         thread_ip_inc(p);
1728 }
1729
1730 static inline void
1731 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1732 {
1733         struct thread *t = &p->threads[p->thread_id];
1734         struct instruction *ip = t->ip;
1735
1736         __instr_hdr_extract3_exec(p, t, ip);
1737
1738         /* Thread. */
1739         thread_ip_inc(p);
1740 }
1741
1742 static inline void
1743 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1744 {
1745         struct thread *t = &p->threads[p->thread_id];
1746         struct instruction *ip = t->ip;
1747
1748         __instr_hdr_extract4_exec(p, t, ip);
1749
1750         /* Thread. */
1751         thread_ip_inc(p);
1752 }
1753
1754 static inline void
1755 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1756 {
1757         struct thread *t = &p->threads[p->thread_id];
1758         struct instruction *ip = t->ip;
1759
1760         __instr_hdr_extract5_exec(p, t, ip);
1761
1762         /* Thread. */
1763         thread_ip_inc(p);
1764 }
1765
1766 static inline void
1767 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1768 {
1769         struct thread *t = &p->threads[p->thread_id];
1770         struct instruction *ip = t->ip;
1771
1772         __instr_hdr_extract6_exec(p, t, ip);
1773
1774         /* Thread. */
1775         thread_ip_inc(p);
1776 }
1777
1778 static inline void
1779 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1780 {
1781         struct thread *t = &p->threads[p->thread_id];
1782         struct instruction *ip = t->ip;
1783
1784         __instr_hdr_extract7_exec(p, t, ip);
1785
1786         /* Thread. */
1787         thread_ip_inc(p);
1788 }
1789
1790 static inline void
1791 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1792 {
1793         struct thread *t = &p->threads[p->thread_id];
1794         struct instruction *ip = t->ip;
1795
1796         __instr_hdr_extract8_exec(p, t, ip);
1797
1798         /* Thread. */
1799         thread_ip_inc(p);
1800 }
1801
1802 static inline void
1803 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1804 {
1805         struct thread *t = &p->threads[p->thread_id];
1806         struct instruction *ip = t->ip;
1807
1808         __instr_hdr_extract_m_exec(p, t, ip);
1809
1810         /* Thread. */
1811         thread_ip_inc(p);
1812 }
1813
1814 static inline void
1815 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1816 {
1817         struct thread *t = &p->threads[p->thread_id];
1818         struct instruction *ip = t->ip;
1819
1820         __instr_hdr_lookahead_exec(p, t, ip);
1821
1822         /* Thread. */
1823         thread_ip_inc(p);
1824 }
1825
1826 /*
1827  * emit.
1828  */
1829 static int
1830 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1831                          struct action *action __rte_unused,
1832                          char **tokens,
1833                          int n_tokens,
1834                          struct instruction *instr,
1835                          struct instruction_data *data __rte_unused)
1836 {
1837         struct header *h;
1838
1839         CHECK(n_tokens == 2, EINVAL);
1840
1841         h = header_parse(p, tokens[1]);
1842         CHECK(h, EINVAL);
1843
1844         instr->type = INSTR_HDR_EMIT;
1845         instr->io.hdr.header_id[0] = h->id;
1846         instr->io.hdr.struct_id[0] = h->struct_id;
1847         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1848         return 0;
1849 }
1850
1851 static inline void
1852 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1853 {
1854         struct thread *t = &p->threads[p->thread_id];
1855         struct instruction *ip = t->ip;
1856
1857         __instr_hdr_emit_exec(p, t, ip);
1858
1859         /* Thread. */
1860         thread_ip_inc(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit2_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit3_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit4_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit5_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 static inline void
1929 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1930 {
1931         struct thread *t = &p->threads[p->thread_id];
1932         struct instruction *ip = t->ip;
1933
1934         __instr_hdr_emit6_tx_exec(p, t, ip);
1935
1936         /* Thread. */
1937         thread_ip_reset(p, t);
1938         instr_rx_exec(p);
1939 }
1940
1941 static inline void
1942 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1943 {
1944         struct thread *t = &p->threads[p->thread_id];
1945         struct instruction *ip = t->ip;
1946
1947         __instr_hdr_emit7_tx_exec(p, t, ip);
1948
1949         /* Thread. */
1950         thread_ip_reset(p, t);
1951         instr_rx_exec(p);
1952 }
1953
1954 static inline void
1955 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1956 {
1957         struct thread *t = &p->threads[p->thread_id];
1958         struct instruction *ip = t->ip;
1959
1960         __instr_hdr_emit8_tx_exec(p, t, ip);
1961
1962         /* Thread. */
1963         thread_ip_reset(p, t);
1964         instr_rx_exec(p);
1965 }
1966
1967 /*
1968  * validate.
1969  */
1970 static int
1971 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1972                              struct action *action __rte_unused,
1973                              char **tokens,
1974                              int n_tokens,
1975                              struct instruction *instr,
1976                              struct instruction_data *data __rte_unused)
1977 {
1978         struct header *h;
1979
1980         CHECK(n_tokens == 2, EINVAL);
1981
1982         h = header_parse(p, tokens[1]);
1983         CHECK(h, EINVAL);
1984
1985         instr->type = INSTR_HDR_VALIDATE;
1986         instr->valid.header_id = h->id;
1987         return 0;
1988 }
1989
1990 static inline void
1991 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1992 {
1993         struct thread *t = &p->threads[p->thread_id];
1994         struct instruction *ip = t->ip;
1995
1996         __instr_hdr_validate_exec(p, t, ip);
1997
1998         /* Thread. */
1999         thread_ip_inc(p);
2000 }
2001
2002 /*
2003  * invalidate.
2004  */
2005 static int
2006 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2007                                struct action *action __rte_unused,
2008                                char **tokens,
2009                                int n_tokens,
2010                                struct instruction *instr,
2011                                struct instruction_data *data __rte_unused)
2012 {
2013         struct header *h;
2014
2015         CHECK(n_tokens == 2, EINVAL);
2016
2017         h = header_parse(p, tokens[1]);
2018         CHECK(h, EINVAL);
2019
2020         instr->type = INSTR_HDR_INVALIDATE;
2021         instr->valid.header_id = h->id;
2022         return 0;
2023 }
2024
2025 static inline void
2026 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2027 {
2028         struct thread *t = &p->threads[p->thread_id];
2029         struct instruction *ip = t->ip;
2030
2031         __instr_hdr_invalidate_exec(p, t, ip);
2032
2033         /* Thread. */
2034         thread_ip_inc(p);
2035 }
2036
2037 /*
2038  * table.
2039  */
2040 static struct table *
2041 table_find(struct rte_swx_pipeline *p, const char *name);
2042
2043 static struct selector *
2044 selector_find(struct rte_swx_pipeline *p, const char *name);
2045
2046 static struct learner *
2047 learner_find(struct rte_swx_pipeline *p, const char *name);
2048
2049 static int
2050 instr_table_translate(struct rte_swx_pipeline *p,
2051                       struct action *action,
2052                       char **tokens,
2053                       int n_tokens,
2054                       struct instruction *instr,
2055                       struct instruction_data *data __rte_unused)
2056 {
2057         struct table *t;
2058         struct selector *s;
2059         struct learner *l;
2060
2061         CHECK(!action, EINVAL);
2062         CHECK(n_tokens == 2, EINVAL);
2063
2064         t = table_find(p, tokens[1]);
2065         if (t) {
2066                 instr->type = INSTR_TABLE;
2067                 instr->table.table_id = t->id;
2068                 return 0;
2069         }
2070
2071         s = selector_find(p, tokens[1]);
2072         if (s) {
2073                 instr->type = INSTR_SELECTOR;
2074                 instr->table.table_id = s->id;
2075                 return 0;
2076         }
2077
2078         l = learner_find(p, tokens[1]);
2079         if (l) {
2080                 instr->type = INSTR_LEARNER;
2081                 instr->table.table_id = l->id;
2082                 return 0;
2083         }
2084
2085         CHECK(0, EINVAL);
2086 }
2087
2088 static inline void
2089 instr_table_exec(struct rte_swx_pipeline *p)
2090 {
2091         struct thread *t = &p->threads[p->thread_id];
2092         struct instruction *ip = t->ip;
2093         uint32_t table_id = ip->table.table_id;
2094         struct rte_swx_table_state *ts = &t->table_state[table_id];
2095         struct table_runtime *table = &t->tables[table_id];
2096         struct table_statistics *stats = &p->table_stats[table_id];
2097         uint64_t action_id, n_pkts_hit, n_pkts_action;
2098         uint8_t *action_data;
2099         int done, hit;
2100
2101         /* Table. */
2102         done = table->func(ts->obj,
2103                            table->mailbox,
2104                            table->key,
2105                            &action_id,
2106                            &action_data,
2107                            &hit);
2108         if (!done) {
2109                 /* Thread. */
2110                 TRACE("[Thread %2u] table %u (not finalized)\n",
2111                       p->thread_id,
2112                       table_id);
2113
2114                 thread_yield(p);
2115                 return;
2116         }
2117
2118         action_id = hit ? action_id : ts->default_action_id;
2119         action_data = hit ? action_data : ts->default_action_data;
2120         n_pkts_hit = stats->n_pkts_hit[hit];
2121         n_pkts_action = stats->n_pkts_action[action_id];
2122
2123         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2124               p->thread_id,
2125               table_id,
2126               hit ? "hit" : "miss",
2127               (uint32_t)action_id);
2128
2129         t->action_id = action_id;
2130         t->structs[0] = action_data;
2131         t->hit = hit;
2132         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2133         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2134
2135         /* Thread. */
2136         thread_ip_action_call(p, t, action_id);
2137 }
2138
2139 static inline void
2140 instr_table_af_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t table_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[table_id];
2146         struct table_runtime *table = &t->tables[table_id];
2147         struct table_statistics *stats = &p->table_stats[table_id];
2148         uint64_t action_id, n_pkts_hit, n_pkts_action;
2149         uint8_t *action_data;
2150         action_func_t action_func;
2151         int done, hit;
2152
2153         /* Table. */
2154         done = table->func(ts->obj,
2155                            table->mailbox,
2156                            table->key,
2157                            &action_id,
2158                            &action_data,
2159                            &hit);
2160         if (!done) {
2161                 /* Thread. */
2162                 TRACE("[Thread %2u] table %u (not finalized)\n",
2163                       p->thread_id,
2164                       table_id);
2165
2166                 thread_yield(p);
2167                 return;
2168         }
2169
2170         action_id = hit ? action_id : ts->default_action_id;
2171         action_data = hit ? action_data : ts->default_action_data;
2172         action_func = p->action_funcs[action_id];
2173         n_pkts_hit = stats->n_pkts_hit[hit];
2174         n_pkts_action = stats->n_pkts_action[action_id];
2175
2176         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2177               p->thread_id,
2178               table_id,
2179               hit ? "hit" : "miss",
2180               (uint32_t)action_id);
2181
2182         t->action_id = action_id;
2183         t->structs[0] = action_data;
2184         t->hit = hit;
2185         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2186         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2187
2188         /* Thread. */
2189         thread_ip_inc(p);
2190
2191         /* Action. */
2192         action_func(p);
2193 }
2194
2195 static inline void
2196 instr_selector_exec(struct rte_swx_pipeline *p)
2197 {
2198         struct thread *t = &p->threads[p->thread_id];
2199         struct instruction *ip = t->ip;
2200         uint32_t selector_id = ip->table.table_id;
2201         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2202         struct selector_runtime *selector = &t->selectors[selector_id];
2203         struct selector_statistics *stats = &p->selector_stats[selector_id];
2204         uint64_t n_pkts = stats->n_pkts;
2205         int done;
2206
2207         /* Table. */
2208         done = rte_swx_table_selector_select(ts->obj,
2209                            selector->mailbox,
2210                            selector->group_id_buffer,
2211                            selector->selector_buffer,
2212                            selector->member_id_buffer);
2213         if (!done) {
2214                 /* Thread. */
2215                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2216                       p->thread_id,
2217                       selector_id);
2218
2219                 thread_yield(p);
2220                 return;
2221         }
2222
2223
2224         TRACE("[Thread %2u] selector %u\n",
2225               p->thread_id,
2226               selector_id);
2227
2228         stats->n_pkts = n_pkts + 1;
2229
2230         /* Thread. */
2231         thread_ip_inc(p);
2232 }
2233
2234 static inline void
2235 instr_learner_exec(struct rte_swx_pipeline *p)
2236 {
2237         struct thread *t = &p->threads[p->thread_id];
2238         struct instruction *ip = t->ip;
2239         uint32_t learner_id = ip->table.table_id;
2240         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2241                 p->n_selectors + learner_id];
2242         struct learner_runtime *l = &t->learners[learner_id];
2243         struct learner_statistics *stats = &p->learner_stats[learner_id];
2244         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2245         uint8_t *action_data;
2246         int done, hit;
2247
2248         /* Table. */
2249         time = rte_get_tsc_cycles();
2250
2251         done = rte_swx_table_learner_lookup(ts->obj,
2252                                             l->mailbox,
2253                                             time,
2254                                             l->key,
2255                                             &action_id,
2256                                             &action_data,
2257                                             &hit);
2258         if (!done) {
2259                 /* Thread. */
2260                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2261                       p->thread_id,
2262                       learner_id);
2263
2264                 thread_yield(p);
2265                 return;
2266         }
2267
2268         action_id = hit ? action_id : ts->default_action_id;
2269         action_data = hit ? action_data : ts->default_action_data;
2270         n_pkts_hit = stats->n_pkts_hit[hit];
2271         n_pkts_action = stats->n_pkts_action[action_id];
2272
2273         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2274               p->thread_id,
2275               learner_id,
2276               hit ? "hit" : "miss",
2277               (uint32_t)action_id);
2278
2279         t->action_id = action_id;
2280         t->structs[0] = action_data;
2281         t->hit = hit;
2282         t->learner_id = learner_id;
2283         t->time = time;
2284         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2285         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2286
2287         /* Thread. */
2288         thread_ip_action_call(p, t, action_id);
2289 }
2290
2291 static inline void
2292 instr_learner_af_exec(struct rte_swx_pipeline *p)
2293 {
2294         struct thread *t = &p->threads[p->thread_id];
2295         struct instruction *ip = t->ip;
2296         uint32_t learner_id = ip->table.table_id;
2297         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2298                 p->n_selectors + learner_id];
2299         struct learner_runtime *l = &t->learners[learner_id];
2300         struct learner_statistics *stats = &p->learner_stats[learner_id];
2301         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2302         uint8_t *action_data;
2303         action_func_t action_func;
2304         int done, hit;
2305
2306         /* Table. */
2307         time = rte_get_tsc_cycles();
2308
2309         done = rte_swx_table_learner_lookup(ts->obj,
2310                                             l->mailbox,
2311                                             time,
2312                                             l->key,
2313                                             &action_id,
2314                                             &action_data,
2315                                             &hit);
2316         if (!done) {
2317                 /* Thread. */
2318                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2319                       p->thread_id,
2320                       learner_id);
2321
2322                 thread_yield(p);
2323                 return;
2324         }
2325
2326         action_id = hit ? action_id : ts->default_action_id;
2327         action_data = hit ? action_data : ts->default_action_data;
2328         action_func = p->action_funcs[action_id];
2329         n_pkts_hit = stats->n_pkts_hit[hit];
2330         n_pkts_action = stats->n_pkts_action[action_id];
2331
2332         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2333               p->thread_id,
2334               learner_id,
2335               hit ? "hit" : "miss",
2336               (uint32_t)action_id);
2337
2338         t->action_id = action_id;
2339         t->structs[0] = action_data;
2340         t->hit = hit;
2341         t->learner_id = learner_id;
2342         t->time = time;
2343         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2344         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2345
2346         /* Thread. */
2347         thread_ip_action_call(p, t, action_id);
2348
2349         /* Action */
2350         action_func(p);
2351 }
2352
2353 /*
2354  * learn.
2355  */
2356 static struct action *
2357 action_find(struct rte_swx_pipeline *p, const char *name);
2358
2359 static int
2360 action_has_nbo_args(struct action *a);
2361
2362 static int
2363 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2364
2365 static int
2366 instr_learn_translate(struct rte_swx_pipeline *p,
2367                       struct action *action,
2368                       char **tokens,
2369                       int n_tokens,
2370                       struct instruction *instr,
2371                       struct instruction_data *data __rte_unused)
2372 {
2373         struct action *a;
2374         const char *mf_name;
2375         uint32_t mf_offset = 0;
2376
2377         CHECK(action, EINVAL);
2378         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2379
2380         a = action_find(p, tokens[1]);
2381         CHECK(a, EINVAL);
2382         CHECK(!action_has_nbo_args(a), EINVAL);
2383
2384         mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2385         CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2386
2387         if (mf_name) {
2388                 struct field *mf;
2389
2390                 mf = metadata_field_parse(p, mf_name);
2391                 CHECK(mf, EINVAL);
2392
2393                 mf_offset = mf->offset / 8;
2394         }
2395
2396         instr->type = INSTR_LEARNER_LEARN;
2397         instr->learn.action_id = a->id;
2398         instr->learn.mf_offset = mf_offset;
2399
2400         return 0;
2401 }
2402
2403 static inline void
2404 instr_learn_exec(struct rte_swx_pipeline *p)
2405 {
2406         struct thread *t = &p->threads[p->thread_id];
2407         struct instruction *ip = t->ip;
2408
2409         __instr_learn_exec(p, t, ip);
2410
2411         /* Thread. */
2412         thread_ip_inc(p);
2413 }
2414
2415 /*
2416  * forget.
2417  */
2418 static int
2419 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2420                        struct action *action,
2421                        char **tokens __rte_unused,
2422                        int n_tokens,
2423                        struct instruction *instr,
2424                        struct instruction_data *data __rte_unused)
2425 {
2426         CHECK(action, EINVAL);
2427         CHECK(n_tokens == 1, EINVAL);
2428
2429         instr->type = INSTR_LEARNER_FORGET;
2430
2431         return 0;
2432 }
2433
2434 static inline void
2435 instr_forget_exec(struct rte_swx_pipeline *p)
2436 {
2437         struct thread *t = &p->threads[p->thread_id];
2438         struct instruction *ip = t->ip;
2439
2440         __instr_forget_exec(p, t, ip);
2441
2442         /* Thread. */
2443         thread_ip_inc(p);
2444 }
2445
2446 /*
2447  * extern.
2448  */
2449 static int
2450 instr_extern_translate(struct rte_swx_pipeline *p,
2451                        struct action *action __rte_unused,
2452                        char **tokens,
2453                        int n_tokens,
2454                        struct instruction *instr,
2455                        struct instruction_data *data __rte_unused)
2456 {
2457         char *token = tokens[1];
2458
2459         CHECK(n_tokens == 2, EINVAL);
2460
2461         if (token[0] == 'e') {
2462                 struct extern_obj *obj;
2463                 struct extern_type_member_func *func;
2464
2465                 func = extern_obj_member_func_parse(p, token, &obj);
2466                 CHECK(func, EINVAL);
2467
2468                 instr->type = INSTR_EXTERN_OBJ;
2469                 instr->ext_obj.ext_obj_id = obj->id;
2470                 instr->ext_obj.func_id = func->id;
2471
2472                 return 0;
2473         }
2474
2475         if (token[0] == 'f') {
2476                 struct extern_func *func;
2477
2478                 func = extern_func_parse(p, token);
2479                 CHECK(func, EINVAL);
2480
2481                 instr->type = INSTR_EXTERN_FUNC;
2482                 instr->ext_func.ext_func_id = func->id;
2483
2484                 return 0;
2485         }
2486
2487         CHECK(0, EINVAL);
2488 }
2489
2490 static inline void
2491 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2492 {
2493         struct thread *t = &p->threads[p->thread_id];
2494         struct instruction *ip = t->ip;
2495         uint32_t done;
2496
2497         /* Extern object member function execute. */
2498         done = __instr_extern_obj_exec(p, t, ip);
2499
2500         /* Thread. */
2501         thread_ip_inc_cond(t, done);
2502         thread_yield_cond(p, done ^ 1);
2503 }
2504
2505 static inline void
2506 instr_extern_func_exec(struct rte_swx_pipeline *p)
2507 {
2508         struct thread *t = &p->threads[p->thread_id];
2509         struct instruction *ip = t->ip;
2510         uint32_t done;
2511
2512         /* Extern function execute. */
2513         done = __instr_extern_func_exec(p, t, ip);
2514
2515         /* Thread. */
2516         thread_ip_inc_cond(t, done);
2517         thread_yield_cond(p, done ^ 1);
2518 }
2519
2520 /*
2521  * mov.
2522  */
2523 static int
2524 instr_mov_translate(struct rte_swx_pipeline *p,
2525                     struct action *action,
2526                     char **tokens,
2527                     int n_tokens,
2528                     struct instruction *instr,
2529                     struct instruction_data *data __rte_unused)
2530 {
2531         char *dst = tokens[1], *src = tokens[2];
2532         struct field *fdst, *fsrc;
2533         uint64_t src_val;
2534         uint32_t dst_struct_id = 0, src_struct_id = 0;
2535
2536         CHECK(n_tokens == 3, EINVAL);
2537
2538         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2539         CHECK(fdst, EINVAL);
2540         CHECK(!fdst->var_size, EINVAL);
2541
2542         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2543         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2544         if (fsrc) {
2545                 CHECK(!fsrc->var_size, EINVAL);
2546
2547                 instr->type = INSTR_MOV;
2548                 if (dst[0] != 'h' && src[0] == 'h')
2549                         instr->type = INSTR_MOV_MH;
2550                 if (dst[0] == 'h' && src[0] != 'h')
2551                         instr->type = INSTR_MOV_HM;
2552                 if (dst[0] == 'h' && src[0] == 'h')
2553                         instr->type = INSTR_MOV_HH;
2554
2555                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2556                 instr->mov.dst.n_bits = fdst->n_bits;
2557                 instr->mov.dst.offset = fdst->offset / 8;
2558                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2559                 instr->mov.src.n_bits = fsrc->n_bits;
2560                 instr->mov.src.offset = fsrc->offset / 8;
2561                 return 0;
2562         }
2563
2564         /* MOV_I. */
2565         src_val = strtoull(src, &src, 0);
2566         CHECK(!src[0], EINVAL);
2567
2568         if (dst[0] == 'h')
2569                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2570
2571         instr->type = INSTR_MOV_I;
2572         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2573         instr->mov.dst.n_bits = fdst->n_bits;
2574         instr->mov.dst.offset = fdst->offset / 8;
2575         instr->mov.src_val = src_val;
2576         return 0;
2577 }
2578
2579 static inline void
2580 instr_mov_exec(struct rte_swx_pipeline *p)
2581 {
2582         struct thread *t = &p->threads[p->thread_id];
2583         struct instruction *ip = t->ip;
2584
2585         __instr_mov_exec(p, t, ip);
2586
2587         /* Thread. */
2588         thread_ip_inc(p);
2589 }
2590
2591 static inline void
2592 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2593 {
2594         struct thread *t = &p->threads[p->thread_id];
2595         struct instruction *ip = t->ip;
2596
2597         __instr_mov_mh_exec(p, t, ip);
2598
2599         /* Thread. */
2600         thread_ip_inc(p);
2601 }
2602
2603 static inline void
2604 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2605 {
2606         struct thread *t = &p->threads[p->thread_id];
2607         struct instruction *ip = t->ip;
2608
2609         __instr_mov_hm_exec(p, t, ip);
2610
2611         /* Thread. */
2612         thread_ip_inc(p);
2613 }
2614
2615 static inline void
2616 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2617 {
2618         struct thread *t = &p->threads[p->thread_id];
2619         struct instruction *ip = t->ip;
2620
2621         __instr_mov_hh_exec(p, t, ip);
2622
2623         /* Thread. */
2624         thread_ip_inc(p);
2625 }
2626
2627 static inline void
2628 instr_mov_i_exec(struct rte_swx_pipeline *p)
2629 {
2630         struct thread *t = &p->threads[p->thread_id];
2631         struct instruction *ip = t->ip;
2632
2633         __instr_mov_i_exec(p, t, ip);
2634
2635         /* Thread. */
2636         thread_ip_inc(p);
2637 }
2638
2639 /*
2640  * dma.
2641  */
2642 static inline void
2643 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2644 {
2645         struct thread *t = &p->threads[p->thread_id];
2646         struct instruction *ip = t->ip;
2647
2648         __instr_dma_ht_exec(p, t, ip);
2649
2650         /* Thread. */
2651         thread_ip_inc(p);
2652 }
2653
2654 static inline void
2655 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2656 {
2657         struct thread *t = &p->threads[p->thread_id];
2658         struct instruction *ip = t->ip;
2659
2660         __instr_dma_ht2_exec(p, t, ip);
2661
2662         /* Thread. */
2663         thread_ip_inc(p);
2664 }
2665
2666 static inline void
2667 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2668 {
2669         struct thread *t = &p->threads[p->thread_id];
2670         struct instruction *ip = t->ip;
2671
2672         __instr_dma_ht3_exec(p, t, ip);
2673
2674         /* Thread. */
2675         thread_ip_inc(p);
2676 }
2677
2678 static inline void
2679 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2680 {
2681         struct thread *t = &p->threads[p->thread_id];
2682         struct instruction *ip = t->ip;
2683
2684         __instr_dma_ht4_exec(p, t, ip);
2685
2686         /* Thread. */
2687         thread_ip_inc(p);
2688 }
2689
2690 static inline void
2691 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2692 {
2693         struct thread *t = &p->threads[p->thread_id];
2694         struct instruction *ip = t->ip;
2695
2696         __instr_dma_ht5_exec(p, t, ip);
2697
2698         /* Thread. */
2699         thread_ip_inc(p);
2700 }
2701
2702 static inline void
2703 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2704 {
2705         struct thread *t = &p->threads[p->thread_id];
2706         struct instruction *ip = t->ip;
2707
2708         __instr_dma_ht6_exec(p, t, ip);
2709
2710         /* Thread. */
2711         thread_ip_inc(p);
2712 }
2713
2714 static inline void
2715 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2716 {
2717         struct thread *t = &p->threads[p->thread_id];
2718         struct instruction *ip = t->ip;
2719
2720         __instr_dma_ht7_exec(p, t, ip);
2721
2722         /* Thread. */
2723         thread_ip_inc(p);
2724 }
2725
2726 static inline void
2727 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2728 {
2729         struct thread *t = &p->threads[p->thread_id];
2730         struct instruction *ip = t->ip;
2731
2732         __instr_dma_ht8_exec(p, t, ip);
2733
2734         /* Thread. */
2735         thread_ip_inc(p);
2736 }
2737
2738 /*
2739  * alu.
2740  */
2741 static int
2742 instr_alu_add_translate(struct rte_swx_pipeline *p,
2743                         struct action *action,
2744                         char **tokens,
2745                         int n_tokens,
2746                         struct instruction *instr,
2747                         struct instruction_data *data __rte_unused)
2748 {
2749         char *dst = tokens[1], *src = tokens[2];
2750         struct field *fdst, *fsrc;
2751         uint64_t src_val;
2752         uint32_t dst_struct_id = 0, src_struct_id = 0;
2753
2754         CHECK(n_tokens == 3, EINVAL);
2755
2756         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2757         CHECK(fdst, EINVAL);
2758         CHECK(!fdst->var_size, EINVAL);
2759
2760         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2761         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2762         if (fsrc) {
2763                 CHECK(!fsrc->var_size, EINVAL);
2764
2765                 instr->type = INSTR_ALU_ADD;
2766                 if (dst[0] == 'h' && src[0] != 'h')
2767                         instr->type = INSTR_ALU_ADD_HM;
2768                 if (dst[0] != 'h' && src[0] == 'h')
2769                         instr->type = INSTR_ALU_ADD_MH;
2770                 if (dst[0] == 'h' && src[0] == 'h')
2771                         instr->type = INSTR_ALU_ADD_HH;
2772
2773                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2774                 instr->alu.dst.n_bits = fdst->n_bits;
2775                 instr->alu.dst.offset = fdst->offset / 8;
2776                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2777                 instr->alu.src.n_bits = fsrc->n_bits;
2778                 instr->alu.src.offset = fsrc->offset / 8;
2779                 return 0;
2780         }
2781
2782         /* ADD_MI, ADD_HI. */
2783         src_val = strtoull(src, &src, 0);
2784         CHECK(!src[0], EINVAL);
2785
2786         instr->type = INSTR_ALU_ADD_MI;
2787         if (dst[0] == 'h')
2788                 instr->type = INSTR_ALU_ADD_HI;
2789
2790         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2791         instr->alu.dst.n_bits = fdst->n_bits;
2792         instr->alu.dst.offset = fdst->offset / 8;
2793         instr->alu.src_val = src_val;
2794         return 0;
2795 }
2796
2797 static int
2798 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2799                         struct action *action,
2800                         char **tokens,
2801                         int n_tokens,
2802                         struct instruction *instr,
2803                         struct instruction_data *data __rte_unused)
2804 {
2805         char *dst = tokens[1], *src = tokens[2];
2806         struct field *fdst, *fsrc;
2807         uint64_t src_val;
2808         uint32_t dst_struct_id = 0, src_struct_id = 0;
2809
2810         CHECK(n_tokens == 3, EINVAL);
2811
2812         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2813         CHECK(fdst, EINVAL);
2814         CHECK(!fdst->var_size, EINVAL);
2815
2816         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2817         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2818         if (fsrc) {
2819                 CHECK(!fsrc->var_size, EINVAL);
2820
2821                 instr->type = INSTR_ALU_SUB;
2822                 if (dst[0] == 'h' && src[0] != 'h')
2823                         instr->type = INSTR_ALU_SUB_HM;
2824                 if (dst[0] != 'h' && src[0] == 'h')
2825                         instr->type = INSTR_ALU_SUB_MH;
2826                 if (dst[0] == 'h' && src[0] == 'h')
2827                         instr->type = INSTR_ALU_SUB_HH;
2828
2829                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2830                 instr->alu.dst.n_bits = fdst->n_bits;
2831                 instr->alu.dst.offset = fdst->offset / 8;
2832                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2833                 instr->alu.src.n_bits = fsrc->n_bits;
2834                 instr->alu.src.offset = fsrc->offset / 8;
2835                 return 0;
2836         }
2837
2838         /* SUB_MI, SUB_HI. */
2839         src_val = strtoull(src, &src, 0);
2840         CHECK(!src[0], EINVAL);
2841
2842         instr->type = INSTR_ALU_SUB_MI;
2843         if (dst[0] == 'h')
2844                 instr->type = INSTR_ALU_SUB_HI;
2845
2846         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2847         instr->alu.dst.n_bits = fdst->n_bits;
2848         instr->alu.dst.offset = fdst->offset / 8;
2849         instr->alu.src_val = src_val;
2850         return 0;
2851 }
2852
2853 static int
2854 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2855                           struct action *action __rte_unused,
2856                           char **tokens,
2857                           int n_tokens,
2858                           struct instruction *instr,
2859                           struct instruction_data *data __rte_unused)
2860 {
2861         char *dst = tokens[1], *src = tokens[2];
2862         struct header *hdst, *hsrc;
2863         struct field *fdst, *fsrc;
2864
2865         CHECK(n_tokens == 3, EINVAL);
2866
2867         fdst = header_field_parse(p, dst, &hdst);
2868         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2869         CHECK(!fdst->var_size, EINVAL);
2870
2871         /* CKADD_FIELD. */
2872         fsrc = header_field_parse(p, src, &hsrc);
2873         if (fsrc) {
2874                 CHECK(!fsrc->var_size, EINVAL);
2875
2876                 instr->type = INSTR_ALU_CKADD_FIELD;
2877                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2878                 instr->alu.dst.n_bits = fdst->n_bits;
2879                 instr->alu.dst.offset = fdst->offset / 8;
2880                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2881                 instr->alu.src.n_bits = fsrc->n_bits;
2882                 instr->alu.src.offset = fsrc->offset / 8;
2883                 return 0;
2884         }
2885
2886         /* CKADD_STRUCT, CKADD_STRUCT20. */
2887         hsrc = header_parse(p, src);
2888         CHECK(hsrc, EINVAL);
2889         CHECK(!hsrc->st->var_size, EINVAL);
2890
2891         instr->type = INSTR_ALU_CKADD_STRUCT;
2892         if ((hsrc->st->n_bits / 8) == 20)
2893                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2894
2895         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2896         instr->alu.dst.n_bits = fdst->n_bits;
2897         instr->alu.dst.offset = fdst->offset / 8;
2898         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2899         instr->alu.src.n_bits = hsrc->st->n_bits;
2900         instr->alu.src.offset = 0; /* Unused. */
2901         return 0;
2902 }
2903
2904 static int
2905 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2906                           struct action *action __rte_unused,
2907                           char **tokens,
2908                           int n_tokens,
2909                           struct instruction *instr,
2910                           struct instruction_data *data __rte_unused)
2911 {
2912         char *dst = tokens[1], *src = tokens[2];
2913         struct header *hdst, *hsrc;
2914         struct field *fdst, *fsrc;
2915
2916         CHECK(n_tokens == 3, EINVAL);
2917
2918         fdst = header_field_parse(p, dst, &hdst);
2919         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2920         CHECK(!fdst->var_size, EINVAL);
2921
2922         fsrc = header_field_parse(p, src, &hsrc);
2923         CHECK(fsrc, EINVAL);
2924         CHECK(!fsrc->var_size, EINVAL);
2925
2926         instr->type = INSTR_ALU_CKSUB_FIELD;
2927         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2928         instr->alu.dst.n_bits = fdst->n_bits;
2929         instr->alu.dst.offset = fdst->offset / 8;
2930         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2931         instr->alu.src.n_bits = fsrc->n_bits;
2932         instr->alu.src.offset = fsrc->offset / 8;
2933         return 0;
2934 }
2935
2936 static int
2937 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2938                         struct action *action,
2939                         char **tokens,
2940                         int n_tokens,
2941                         struct instruction *instr,
2942                         struct instruction_data *data __rte_unused)
2943 {
2944         char *dst = tokens[1], *src = tokens[2];
2945         struct field *fdst, *fsrc;
2946         uint64_t src_val;
2947         uint32_t dst_struct_id = 0, src_struct_id = 0;
2948
2949         CHECK(n_tokens == 3, EINVAL);
2950
2951         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2952         CHECK(fdst, EINVAL);
2953         CHECK(!fdst->var_size, EINVAL);
2954
2955         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2956         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2957         if (fsrc) {
2958                 CHECK(!fsrc->var_size, EINVAL);
2959
2960                 instr->type = INSTR_ALU_SHL;
2961                 if (dst[0] == 'h' && src[0] != 'h')
2962                         instr->type = INSTR_ALU_SHL_HM;
2963                 if (dst[0] != 'h' && src[0] == 'h')
2964                         instr->type = INSTR_ALU_SHL_MH;
2965                 if (dst[0] == 'h' && src[0] == 'h')
2966                         instr->type = INSTR_ALU_SHL_HH;
2967
2968                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2969                 instr->alu.dst.n_bits = fdst->n_bits;
2970                 instr->alu.dst.offset = fdst->offset / 8;
2971                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2972                 instr->alu.src.n_bits = fsrc->n_bits;
2973                 instr->alu.src.offset = fsrc->offset / 8;
2974                 return 0;
2975         }
2976
2977         /* SHL_MI, SHL_HI. */
2978         src_val = strtoull(src, &src, 0);
2979         CHECK(!src[0], EINVAL);
2980
2981         instr->type = INSTR_ALU_SHL_MI;
2982         if (dst[0] == 'h')
2983                 instr->type = INSTR_ALU_SHL_HI;
2984
2985         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2986         instr->alu.dst.n_bits = fdst->n_bits;
2987         instr->alu.dst.offset = fdst->offset / 8;
2988         instr->alu.src_val = src_val;
2989         return 0;
2990 }
2991
2992 static int
2993 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2994                         struct action *action,
2995                         char **tokens,
2996                         int n_tokens,
2997                         struct instruction *instr,
2998                         struct instruction_data *data __rte_unused)
2999 {
3000         char *dst = tokens[1], *src = tokens[2];
3001         struct field *fdst, *fsrc;
3002         uint64_t src_val;
3003         uint32_t dst_struct_id = 0, src_struct_id = 0;
3004
3005         CHECK(n_tokens == 3, EINVAL);
3006
3007         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3008         CHECK(fdst, EINVAL);
3009         CHECK(!fdst->var_size, EINVAL);
3010
3011         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3012         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3013         if (fsrc) {
3014                 CHECK(!fsrc->var_size, EINVAL);
3015
3016                 instr->type = INSTR_ALU_SHR;
3017                 if (dst[0] == 'h' && src[0] != 'h')
3018                         instr->type = INSTR_ALU_SHR_HM;
3019                 if (dst[0] != 'h' && src[0] == 'h')
3020                         instr->type = INSTR_ALU_SHR_MH;
3021                 if (dst[0] == 'h' && src[0] == 'h')
3022                         instr->type = INSTR_ALU_SHR_HH;
3023
3024                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3025                 instr->alu.dst.n_bits = fdst->n_bits;
3026                 instr->alu.dst.offset = fdst->offset / 8;
3027                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3028                 instr->alu.src.n_bits = fsrc->n_bits;
3029                 instr->alu.src.offset = fsrc->offset / 8;
3030                 return 0;
3031         }
3032
3033         /* SHR_MI, SHR_HI. */
3034         src_val = strtoull(src, &src, 0);
3035         CHECK(!src[0], EINVAL);
3036
3037         instr->type = INSTR_ALU_SHR_MI;
3038         if (dst[0] == 'h')
3039                 instr->type = INSTR_ALU_SHR_HI;
3040
3041         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3042         instr->alu.dst.n_bits = fdst->n_bits;
3043         instr->alu.dst.offset = fdst->offset / 8;
3044         instr->alu.src_val = src_val;
3045         return 0;
3046 }
3047
3048 static int
3049 instr_alu_and_translate(struct rte_swx_pipeline *p,
3050                         struct action *action,
3051                         char **tokens,
3052                         int n_tokens,
3053                         struct instruction *instr,
3054                         struct instruction_data *data __rte_unused)
3055 {
3056         char *dst = tokens[1], *src = tokens[2];
3057         struct field *fdst, *fsrc;
3058         uint64_t src_val;
3059         uint32_t dst_struct_id = 0, src_struct_id = 0;
3060
3061         CHECK(n_tokens == 3, EINVAL);
3062
3063         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3064         CHECK(fdst, EINVAL);
3065         CHECK(!fdst->var_size, EINVAL);
3066
3067         /* AND, AND_MH, AND_HM, AND_HH. */
3068         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3069         if (fsrc) {
3070                 CHECK(!fsrc->var_size, EINVAL);
3071
3072                 instr->type = INSTR_ALU_AND;
3073                 if (dst[0] != 'h' && src[0] == 'h')
3074                         instr->type = INSTR_ALU_AND_MH;
3075                 if (dst[0] == 'h' && src[0] != 'h')
3076                         instr->type = INSTR_ALU_AND_HM;
3077                 if (dst[0] == 'h' && src[0] == 'h')
3078                         instr->type = INSTR_ALU_AND_HH;
3079
3080                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3081                 instr->alu.dst.n_bits = fdst->n_bits;
3082                 instr->alu.dst.offset = fdst->offset / 8;
3083                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3084                 instr->alu.src.n_bits = fsrc->n_bits;
3085                 instr->alu.src.offset = fsrc->offset / 8;
3086                 return 0;
3087         }
3088
3089         /* AND_I. */
3090         src_val = strtoull(src, &src, 0);
3091         CHECK(!src[0], EINVAL);
3092
3093         if (dst[0] == 'h')
3094                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3095
3096         instr->type = INSTR_ALU_AND_I;
3097         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3098         instr->alu.dst.n_bits = fdst->n_bits;
3099         instr->alu.dst.offset = fdst->offset / 8;
3100         instr->alu.src_val = src_val;
3101         return 0;
3102 }
3103
3104 static int
3105 instr_alu_or_translate(struct rte_swx_pipeline *p,
3106                        struct action *action,
3107                        char **tokens,
3108                        int n_tokens,
3109                        struct instruction *instr,
3110                        struct instruction_data *data __rte_unused)
3111 {
3112         char *dst = tokens[1], *src = tokens[2];
3113         struct field *fdst, *fsrc;
3114         uint64_t src_val;
3115         uint32_t dst_struct_id = 0, src_struct_id = 0;
3116
3117         CHECK(n_tokens == 3, EINVAL);
3118
3119         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3120         CHECK(fdst, EINVAL);
3121         CHECK(!fdst->var_size, EINVAL);
3122
3123         /* OR, OR_MH, OR_HM, OR_HH. */
3124         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3125         if (fsrc) {
3126                 CHECK(!fsrc->var_size, EINVAL);
3127
3128                 instr->type = INSTR_ALU_OR;
3129                 if (dst[0] != 'h' && src[0] == 'h')
3130                         instr->type = INSTR_ALU_OR_MH;
3131                 if (dst[0] == 'h' && src[0] != 'h')
3132                         instr->type = INSTR_ALU_OR_HM;
3133                 if (dst[0] == 'h' && src[0] == 'h')
3134                         instr->type = INSTR_ALU_OR_HH;
3135
3136                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3137                 instr->alu.dst.n_bits = fdst->n_bits;
3138                 instr->alu.dst.offset = fdst->offset / 8;
3139                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3140                 instr->alu.src.n_bits = fsrc->n_bits;
3141                 instr->alu.src.offset = fsrc->offset / 8;
3142                 return 0;
3143         }
3144
3145         /* OR_I. */
3146         src_val = strtoull(src, &src, 0);
3147         CHECK(!src[0], EINVAL);
3148
3149         if (dst[0] == 'h')
3150                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3151
3152         instr->type = INSTR_ALU_OR_I;
3153         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3154         instr->alu.dst.n_bits = fdst->n_bits;
3155         instr->alu.dst.offset = fdst->offset / 8;
3156         instr->alu.src_val = src_val;
3157         return 0;
3158 }
3159
3160 static int
3161 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3162                         struct action *action,
3163                         char **tokens,
3164                         int n_tokens,
3165                         struct instruction *instr,
3166                         struct instruction_data *data __rte_unused)
3167 {
3168         char *dst = tokens[1], *src = tokens[2];
3169         struct field *fdst, *fsrc;
3170         uint64_t src_val;
3171         uint32_t dst_struct_id = 0, src_struct_id = 0;
3172
3173         CHECK(n_tokens == 3, EINVAL);
3174
3175         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3176         CHECK(fdst, EINVAL);
3177         CHECK(!fdst->var_size, EINVAL);
3178
3179         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3180         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3181         if (fsrc) {
3182                 CHECK(!fsrc->var_size, EINVAL);
3183
3184                 instr->type = INSTR_ALU_XOR;
3185                 if (dst[0] != 'h' && src[0] == 'h')
3186                         instr->type = INSTR_ALU_XOR_MH;
3187                 if (dst[0] == 'h' && src[0] != 'h')
3188                         instr->type = INSTR_ALU_XOR_HM;
3189                 if (dst[0] == 'h' && src[0] == 'h')
3190                         instr->type = INSTR_ALU_XOR_HH;
3191
3192                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3193                 instr->alu.dst.n_bits = fdst->n_bits;
3194                 instr->alu.dst.offset = fdst->offset / 8;
3195                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3196                 instr->alu.src.n_bits = fsrc->n_bits;
3197                 instr->alu.src.offset = fsrc->offset / 8;
3198                 return 0;
3199         }
3200
3201         /* XOR_I. */
3202         src_val = strtoull(src, &src, 0);
3203         CHECK(!src[0], EINVAL);
3204
3205         if (dst[0] == 'h')
3206                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3207
3208         instr->type = INSTR_ALU_XOR_I;
3209         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3210         instr->alu.dst.n_bits = fdst->n_bits;
3211         instr->alu.dst.offset = fdst->offset / 8;
3212         instr->alu.src_val = src_val;
3213         return 0;
3214 }
3215
3216 static inline void
3217 instr_alu_add_exec(struct rte_swx_pipeline *p)
3218 {
3219         struct thread *t = &p->threads[p->thread_id];
3220         struct instruction *ip = t->ip;
3221
3222         /* Structs */
3223         __instr_alu_add_exec(p, t, ip);
3224
3225         /* Thread. */
3226         thread_ip_inc(p);
3227 }
3228
3229 static inline void
3230 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3231 {
3232         struct thread *t = &p->threads[p->thread_id];
3233         struct instruction *ip = t->ip;
3234
3235         /* Structs. */
3236         __instr_alu_add_mh_exec(p, t, ip);
3237
3238         /* Thread. */
3239         thread_ip_inc(p);
3240 }
3241
3242 static inline void
3243 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3244 {
3245         struct thread *t = &p->threads[p->thread_id];
3246         struct instruction *ip = t->ip;
3247
3248         /* Structs. */
3249         __instr_alu_add_hm_exec(p, t, ip);
3250
3251         /* Thread. */
3252         thread_ip_inc(p);
3253 }
3254
3255 static inline void
3256 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3257 {
3258         struct thread *t = &p->threads[p->thread_id];
3259         struct instruction *ip = t->ip;
3260
3261         /* Structs. */
3262         __instr_alu_add_hh_exec(p, t, ip);
3263
3264         /* Thread. */
3265         thread_ip_inc(p);
3266 }
3267
3268 static inline void
3269 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3270 {
3271         struct thread *t = &p->threads[p->thread_id];
3272         struct instruction *ip = t->ip;
3273
3274         /* Structs. */
3275         __instr_alu_add_mi_exec(p, t, ip);
3276
3277         /* Thread. */
3278         thread_ip_inc(p);
3279 }
3280
3281 static inline void
3282 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3283 {
3284         struct thread *t = &p->threads[p->thread_id];
3285         struct instruction *ip = t->ip;
3286
3287         /* Structs. */
3288         __instr_alu_add_hi_exec(p, t, ip);
3289
3290         /* Thread. */
3291         thread_ip_inc(p);
3292 }
3293
3294 static inline void
3295 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3296 {
3297         struct thread *t = &p->threads[p->thread_id];
3298         struct instruction *ip = t->ip;
3299
3300         /* Structs. */
3301         __instr_alu_sub_exec(p, t, ip);
3302
3303         /* Thread. */
3304         thread_ip_inc(p);
3305 }
3306
3307 static inline void
3308 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3309 {
3310         struct thread *t = &p->threads[p->thread_id];
3311         struct instruction *ip = t->ip;
3312
3313         /* Structs. */
3314         __instr_alu_sub_mh_exec(p, t, ip);
3315
3316         /* Thread. */
3317         thread_ip_inc(p);
3318 }
3319
3320 static inline void
3321 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3322 {
3323         struct thread *t = &p->threads[p->thread_id];
3324         struct instruction *ip = t->ip;
3325
3326         /* Structs. */
3327         __instr_alu_sub_hm_exec(p, t, ip);
3328
3329         /* Thread. */
3330         thread_ip_inc(p);
3331 }
3332
3333 static inline void
3334 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3335 {
3336         struct thread *t = &p->threads[p->thread_id];
3337         struct instruction *ip = t->ip;
3338
3339         /* Structs. */
3340         __instr_alu_sub_hh_exec(p, t, ip);
3341
3342         /* Thread. */
3343         thread_ip_inc(p);
3344 }
3345
3346 static inline void
3347 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3348 {
3349         struct thread *t = &p->threads[p->thread_id];
3350         struct instruction *ip = t->ip;
3351
3352         /* Structs. */
3353         __instr_alu_sub_mi_exec(p, t, ip);
3354
3355         /* Thread. */
3356         thread_ip_inc(p);
3357 }
3358
3359 static inline void
3360 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3361 {
3362         struct thread *t = &p->threads[p->thread_id];
3363         struct instruction *ip = t->ip;
3364
3365         /* Structs. */
3366         __instr_alu_sub_hi_exec(p, t, ip);
3367
3368         /* Thread. */
3369         thread_ip_inc(p);
3370 }
3371
3372 static inline void
3373 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3374 {
3375         struct thread *t = &p->threads[p->thread_id];
3376         struct instruction *ip = t->ip;
3377
3378         /* Structs. */
3379         __instr_alu_shl_exec(p, t, ip);
3380
3381         /* Thread. */
3382         thread_ip_inc(p);
3383 }
3384
3385 static inline void
3386 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3387 {
3388         struct thread *t = &p->threads[p->thread_id];
3389         struct instruction *ip = t->ip;
3390
3391         /* Structs. */
3392         __instr_alu_shl_mh_exec(p, t, ip);
3393
3394         /* Thread. */
3395         thread_ip_inc(p);
3396 }
3397
3398 static inline void
3399 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3400 {
3401         struct thread *t = &p->threads[p->thread_id];
3402         struct instruction *ip = t->ip;
3403
3404         /* Structs. */
3405         __instr_alu_shl_hm_exec(p, t, ip);
3406
3407         /* Thread. */
3408         thread_ip_inc(p);
3409 }
3410
3411 static inline void
3412 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3413 {
3414         struct thread *t = &p->threads[p->thread_id];
3415         struct instruction *ip = t->ip;
3416
3417         /* Structs. */
3418         __instr_alu_shl_hh_exec(p, t, ip);
3419
3420         /* Thread. */
3421         thread_ip_inc(p);
3422 }
3423
3424 static inline void
3425 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3426 {
3427         struct thread *t = &p->threads[p->thread_id];
3428         struct instruction *ip = t->ip;
3429
3430         /* Structs. */
3431         __instr_alu_shl_mi_exec(p, t, ip);
3432
3433         /* Thread. */
3434         thread_ip_inc(p);
3435 }
3436
3437 static inline void
3438 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3439 {
3440         struct thread *t = &p->threads[p->thread_id];
3441         struct instruction *ip = t->ip;
3442
3443         /* Structs. */
3444         __instr_alu_shl_hi_exec(p, t, ip);
3445
3446         /* Thread. */
3447         thread_ip_inc(p);
3448 }
3449
3450 static inline void
3451 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3452 {
3453         struct thread *t = &p->threads[p->thread_id];
3454         struct instruction *ip = t->ip;
3455
3456         /* Structs. */
3457         __instr_alu_shr_exec(p, t, ip);
3458
3459         /* Thread. */
3460         thread_ip_inc(p);
3461 }
3462
3463 static inline void
3464 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3465 {
3466         struct thread *t = &p->threads[p->thread_id];
3467         struct instruction *ip = t->ip;
3468
3469         /* Structs. */
3470         __instr_alu_shr_mh_exec(p, t, ip);
3471
3472         /* Thread. */
3473         thread_ip_inc(p);
3474 }
3475
3476 static inline void
3477 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3478 {
3479         struct thread *t = &p->threads[p->thread_id];
3480         struct instruction *ip = t->ip;
3481
3482         /* Structs. */
3483         __instr_alu_shr_hm_exec(p, t, ip);
3484
3485         /* Thread. */
3486         thread_ip_inc(p);
3487 }
3488
3489 static inline void
3490 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3491 {
3492         struct thread *t = &p->threads[p->thread_id];
3493         struct instruction *ip = t->ip;
3494
3495         /* Structs. */
3496         __instr_alu_shr_hh_exec(p, t, ip);
3497
3498         /* Thread. */
3499         thread_ip_inc(p);
3500 }
3501
3502 static inline void
3503 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3504 {
3505         struct thread *t = &p->threads[p->thread_id];
3506         struct instruction *ip = t->ip;
3507
3508         /* Structs. */
3509         __instr_alu_shr_mi_exec(p, t, ip);
3510
3511         /* Thread. */
3512         thread_ip_inc(p);
3513 }
3514
3515 static inline void
3516 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3517 {
3518         struct thread *t = &p->threads[p->thread_id];
3519         struct instruction *ip = t->ip;
3520
3521         /* Structs. */
3522         __instr_alu_shr_hi_exec(p, t, ip);
3523
3524         /* Thread. */
3525         thread_ip_inc(p);
3526 }
3527
3528 static inline void
3529 instr_alu_and_exec(struct rte_swx_pipeline *p)
3530 {
3531         struct thread *t = &p->threads[p->thread_id];
3532         struct instruction *ip = t->ip;
3533
3534         /* Structs. */
3535         __instr_alu_and_exec(p, t, ip);
3536
3537         /* Thread. */
3538         thread_ip_inc(p);
3539 }
3540
3541 static inline void
3542 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3543 {
3544         struct thread *t = &p->threads[p->thread_id];
3545         struct instruction *ip = t->ip;
3546
3547         /* Structs. */
3548         __instr_alu_and_mh_exec(p, t, ip);
3549
3550         /* Thread. */
3551         thread_ip_inc(p);
3552 }
3553
3554 static inline void
3555 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3556 {
3557         struct thread *t = &p->threads[p->thread_id];
3558         struct instruction *ip = t->ip;
3559
3560         /* Structs. */
3561         __instr_alu_and_hm_exec(p, t, ip);
3562
3563         /* Thread. */
3564         thread_ip_inc(p);
3565 }
3566
3567 static inline void
3568 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3569 {
3570         struct thread *t = &p->threads[p->thread_id];
3571         struct instruction *ip = t->ip;
3572
3573         /* Structs. */
3574         __instr_alu_and_hh_exec(p, t, ip);
3575
3576         /* Thread. */
3577         thread_ip_inc(p);
3578 }
3579
3580 static inline void
3581 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3582 {
3583         struct thread *t = &p->threads[p->thread_id];
3584         struct instruction *ip = t->ip;
3585
3586         /* Structs. */
3587         __instr_alu_and_i_exec(p, t, ip);
3588
3589         /* Thread. */
3590         thread_ip_inc(p);
3591 }
3592
3593 static inline void
3594 instr_alu_or_exec(struct rte_swx_pipeline *p)
3595 {
3596         struct thread *t = &p->threads[p->thread_id];
3597         struct instruction *ip = t->ip;
3598
3599         /* Structs. */
3600         __instr_alu_or_exec(p, t, ip);
3601
3602         /* Thread. */
3603         thread_ip_inc(p);
3604 }
3605
3606 static inline void
3607 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3608 {
3609         struct thread *t = &p->threads[p->thread_id];
3610         struct instruction *ip = t->ip;
3611
3612         /* Structs. */
3613         __instr_alu_or_mh_exec(p, t, ip);
3614
3615         /* Thread. */
3616         thread_ip_inc(p);
3617 }
3618
3619 static inline void
3620 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3621 {
3622         struct thread *t = &p->threads[p->thread_id];
3623         struct instruction *ip = t->ip;
3624
3625         /* Structs. */
3626         __instr_alu_or_hm_exec(p, t, ip);
3627
3628         /* Thread. */
3629         thread_ip_inc(p);
3630 }
3631
3632 static inline void
3633 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3634 {
3635         struct thread *t = &p->threads[p->thread_id];
3636         struct instruction *ip = t->ip;
3637
3638         /* Structs. */
3639         __instr_alu_or_hh_exec(p, t, ip);
3640
3641         /* Thread. */
3642         thread_ip_inc(p);
3643 }
3644
3645 static inline void
3646 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3647 {
3648         struct thread *t = &p->threads[p->thread_id];
3649         struct instruction *ip = t->ip;
3650
3651         /* Structs. */
3652         __instr_alu_or_i_exec(p, t, ip);
3653
3654         /* Thread. */
3655         thread_ip_inc(p);
3656 }
3657
3658 static inline void
3659 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3660 {
3661         struct thread *t = &p->threads[p->thread_id];
3662         struct instruction *ip = t->ip;
3663
3664         /* Structs. */
3665         __instr_alu_xor_exec(p, t, ip);
3666
3667         /* Thread. */
3668         thread_ip_inc(p);
3669 }
3670
3671 static inline void
3672 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3673 {
3674         struct thread *t = &p->threads[p->thread_id];
3675         struct instruction *ip = t->ip;
3676
3677         /* Structs. */
3678         __instr_alu_xor_mh_exec(p, t, ip);
3679
3680         /* Thread. */
3681         thread_ip_inc(p);
3682 }
3683
3684 static inline void
3685 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3686 {
3687         struct thread *t = &p->threads[p->thread_id];
3688         struct instruction *ip = t->ip;
3689
3690         /* Structs. */
3691         __instr_alu_xor_hm_exec(p, t, ip);
3692
3693         /* Thread. */
3694         thread_ip_inc(p);
3695 }
3696
3697 static inline void
3698 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3699 {
3700         struct thread *t = &p->threads[p->thread_id];
3701         struct instruction *ip = t->ip;
3702
3703         /* Structs. */
3704         __instr_alu_xor_hh_exec(p, t, ip);
3705
3706         /* Thread. */
3707         thread_ip_inc(p);
3708 }
3709
3710 static inline void
3711 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3712 {
3713         struct thread *t = &p->threads[p->thread_id];
3714         struct instruction *ip = t->ip;
3715
3716         /* Structs. */
3717         __instr_alu_xor_i_exec(p, t, ip);
3718
3719         /* Thread. */
3720         thread_ip_inc(p);
3721 }
3722
3723 static inline void
3724 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3725 {
3726         struct thread *t = &p->threads[p->thread_id];
3727         struct instruction *ip = t->ip;
3728
3729         /* Structs. */
3730         __instr_alu_ckadd_field_exec(p, t, ip);
3731
3732         /* Thread. */
3733         thread_ip_inc(p);
3734 }
3735
3736 static inline void
3737 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3738 {
3739         struct thread *t = &p->threads[p->thread_id];
3740         struct instruction *ip = t->ip;
3741
3742         /* Structs. */
3743         __instr_alu_cksub_field_exec(p, t, ip);
3744
3745         /* Thread. */
3746         thread_ip_inc(p);
3747 }
3748
3749 static inline void
3750 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3751 {
3752         struct thread *t = &p->threads[p->thread_id];
3753         struct instruction *ip = t->ip;
3754
3755         /* Structs. */
3756         __instr_alu_ckadd_struct20_exec(p, t, ip);
3757
3758         /* Thread. */
3759         thread_ip_inc(p);
3760 }
3761
3762 static inline void
3763 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3764 {
3765         struct thread *t = &p->threads[p->thread_id];
3766         struct instruction *ip = t->ip;
3767
3768         /* Structs. */
3769         __instr_alu_ckadd_struct_exec(p, t, ip);
3770
3771         /* Thread. */
3772         thread_ip_inc(p);
3773 }
3774
3775 /*
3776  * Register array.
3777  */
3778 static struct regarray *
3779 regarray_find(struct rte_swx_pipeline *p, const char *name);
3780
3781 static int
3782 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3783                       struct action *action,
3784                       char **tokens,
3785                       int n_tokens,
3786                       struct instruction *instr,
3787                       struct instruction_data *data __rte_unused)
3788 {
3789         char *regarray = tokens[1], *idx = tokens[2];
3790         struct regarray *r;
3791         struct field *fidx;
3792         uint32_t idx_struct_id, idx_val;
3793
3794         CHECK(n_tokens == 3, EINVAL);
3795
3796         r = regarray_find(p, regarray);
3797         CHECK(r, EINVAL);
3798
3799         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3800         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3801         if (fidx) {
3802                 CHECK(!fidx->var_size, EINVAL);
3803
3804                 instr->type = INSTR_REGPREFETCH_RM;
3805                 if (idx[0] == 'h')
3806                         instr->type = INSTR_REGPREFETCH_RH;
3807
3808                 instr->regarray.regarray_id = r->id;
3809                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3810                 instr->regarray.idx.n_bits = fidx->n_bits;
3811                 instr->regarray.idx.offset = fidx->offset / 8;
3812                 instr->regarray.dstsrc_val = 0; /* Unused. */
3813                 return 0;
3814         }
3815
3816         /* REGPREFETCH_RI. */
3817         idx_val = strtoul(idx, &idx, 0);
3818         CHECK(!idx[0], EINVAL);
3819
3820         instr->type = INSTR_REGPREFETCH_RI;
3821         instr->regarray.regarray_id = r->id;
3822         instr->regarray.idx_val = idx_val;
3823         instr->regarray.dstsrc_val = 0; /* Unused. */
3824         return 0;
3825 }
3826
3827 static int
3828 instr_regrd_translate(struct rte_swx_pipeline *p,
3829                       struct action *action,
3830                       char **tokens,
3831                       int n_tokens,
3832                       struct instruction *instr,
3833                       struct instruction_data *data __rte_unused)
3834 {
3835         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3836         struct regarray *r;
3837         struct field *fdst, *fidx;
3838         uint32_t dst_struct_id, idx_struct_id, idx_val;
3839
3840         CHECK(n_tokens == 4, EINVAL);
3841
3842         r = regarray_find(p, regarray);
3843         CHECK(r, EINVAL);
3844
3845         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3846         CHECK(fdst, EINVAL);
3847         CHECK(!fdst->var_size, EINVAL);
3848
3849         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3850         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3851         if (fidx) {
3852                 CHECK(!fidx->var_size, EINVAL);
3853
3854                 instr->type = INSTR_REGRD_MRM;
3855                 if (dst[0] == 'h' && idx[0] != 'h')
3856                         instr->type = INSTR_REGRD_HRM;
3857                 if (dst[0] != 'h' && idx[0] == 'h')
3858                         instr->type = INSTR_REGRD_MRH;
3859                 if (dst[0] == 'h' && idx[0] == 'h')
3860                         instr->type = INSTR_REGRD_HRH;
3861
3862                 instr->regarray.regarray_id = r->id;
3863                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3864                 instr->regarray.idx.n_bits = fidx->n_bits;
3865                 instr->regarray.idx.offset = fidx->offset / 8;
3866                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3867                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3868                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3869                 return 0;
3870         }
3871
3872         /* REGRD_MRI, REGRD_HRI. */
3873         idx_val = strtoul(idx, &idx, 0);
3874         CHECK(!idx[0], EINVAL);
3875
3876         instr->type = INSTR_REGRD_MRI;
3877         if (dst[0] == 'h')
3878                 instr->type = INSTR_REGRD_HRI;
3879
3880         instr->regarray.regarray_id = r->id;
3881         instr->regarray.idx_val = idx_val;
3882         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3883         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3884         instr->regarray.dstsrc.offset = fdst->offset / 8;
3885         return 0;
3886 }
3887
3888 static int
3889 instr_regwr_translate(struct rte_swx_pipeline *p,
3890                       struct action *action,
3891                       char **tokens,
3892                       int n_tokens,
3893                       struct instruction *instr,
3894                       struct instruction_data *data __rte_unused)
3895 {
3896         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3897         struct regarray *r;
3898         struct field *fidx, *fsrc;
3899         uint64_t src_val;
3900         uint32_t idx_struct_id, idx_val, src_struct_id;
3901
3902         CHECK(n_tokens == 4, EINVAL);
3903
3904         r = regarray_find(p, regarray);
3905         CHECK(r, EINVAL);
3906
3907         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3908         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3909         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3910         if (fidx && fsrc) {
3911                 CHECK(!fidx->var_size, EINVAL);
3912                 CHECK(!fsrc->var_size, EINVAL);
3913
3914                 instr->type = INSTR_REGWR_RMM;
3915                 if (idx[0] == 'h' && src[0] != 'h')
3916                         instr->type = INSTR_REGWR_RHM;
3917                 if (idx[0] != 'h' && src[0] == 'h')
3918                         instr->type = INSTR_REGWR_RMH;
3919                 if (idx[0] == 'h' && src[0] == 'h')
3920                         instr->type = INSTR_REGWR_RHH;
3921
3922                 instr->regarray.regarray_id = r->id;
3923                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3924                 instr->regarray.idx.n_bits = fidx->n_bits;
3925                 instr->regarray.idx.offset = fidx->offset / 8;
3926                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3927                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3928                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3929                 return 0;
3930         }
3931
3932         /* REGWR_RHI, REGWR_RMI. */
3933         if (fidx && !fsrc) {
3934                 CHECK(!fidx->var_size, EINVAL);
3935
3936                 src_val = strtoull(src, &src, 0);
3937                 CHECK(!src[0], EINVAL);
3938
3939                 instr->type = INSTR_REGWR_RMI;
3940                 if (idx[0] == 'h')
3941                         instr->type = INSTR_REGWR_RHI;
3942
3943                 instr->regarray.regarray_id = r->id;
3944                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3945                 instr->regarray.idx.n_bits = fidx->n_bits;
3946                 instr->regarray.idx.offset = fidx->offset / 8;
3947                 instr->regarray.dstsrc_val = src_val;
3948                 return 0;
3949         }
3950
3951         /* REGWR_RIH, REGWR_RIM. */
3952         if (!fidx && fsrc) {
3953                 idx_val = strtoul(idx, &idx, 0);
3954                 CHECK(!idx[0], EINVAL);
3955
3956                 CHECK(!fsrc->var_size, EINVAL);
3957
3958                 instr->type = INSTR_REGWR_RIM;
3959                 if (src[0] == 'h')
3960                         instr->type = INSTR_REGWR_RIH;
3961
3962                 instr->regarray.regarray_id = r->id;
3963                 instr->regarray.idx_val = idx_val;
3964                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3965                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3966                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3967                 return 0;
3968         }
3969
3970         /* REGWR_RII. */
3971         src_val = strtoull(src, &src, 0);
3972         CHECK(!src[0], EINVAL);
3973
3974         idx_val = strtoul(idx, &idx, 0);
3975         CHECK(!idx[0], EINVAL);
3976
3977         instr->type = INSTR_REGWR_RII;
3978         instr->regarray.idx_val = idx_val;
3979         instr->regarray.dstsrc_val = src_val;
3980
3981         return 0;
3982 }
3983
3984 static int
3985 instr_regadd_translate(struct rte_swx_pipeline *p,
3986                        struct action *action,
3987                        char **tokens,
3988                        int n_tokens,
3989                        struct instruction *instr,
3990                        struct instruction_data *data __rte_unused)
3991 {
3992         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3993         struct regarray *r;
3994         struct field *fidx, *fsrc;
3995         uint64_t src_val;
3996         uint32_t idx_struct_id, idx_val, src_struct_id;
3997
3998         CHECK(n_tokens == 4, EINVAL);
3999
4000         r = regarray_find(p, regarray);
4001         CHECK(r, EINVAL);
4002
4003         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4004         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4005         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4006         if (fidx && fsrc) {
4007                 CHECK(!fidx->var_size, EINVAL);
4008                 CHECK(!fsrc->var_size, EINVAL);
4009
4010                 instr->type = INSTR_REGADD_RMM;
4011                 if (idx[0] == 'h' && src[0] != 'h')
4012                         instr->type = INSTR_REGADD_RHM;
4013                 if (idx[0] != 'h' && src[0] == 'h')
4014                         instr->type = INSTR_REGADD_RMH;
4015                 if (idx[0] == 'h' && src[0] == 'h')
4016                         instr->type = INSTR_REGADD_RHH;
4017
4018                 instr->regarray.regarray_id = r->id;
4019                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4020                 instr->regarray.idx.n_bits = fidx->n_bits;
4021                 instr->regarray.idx.offset = fidx->offset / 8;
4022                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4023                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4024                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4025                 return 0;
4026         }
4027
4028         /* REGADD_RHI, REGADD_RMI. */
4029         if (fidx && !fsrc) {
4030                 CHECK(!fidx->var_size, EINVAL);
4031
4032                 src_val = strtoull(src, &src, 0);
4033                 CHECK(!src[0], EINVAL);
4034
4035                 instr->type = INSTR_REGADD_RMI;
4036                 if (idx[0] == 'h')
4037                         instr->type = INSTR_REGADD_RHI;
4038
4039                 instr->regarray.regarray_id = r->id;
4040                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4041                 instr->regarray.idx.n_bits = fidx->n_bits;
4042                 instr->regarray.idx.offset = fidx->offset / 8;
4043                 instr->regarray.dstsrc_val = src_val;
4044                 return 0;
4045         }
4046
4047         /* REGADD_RIH, REGADD_RIM. */
4048         if (!fidx && fsrc) {
4049                 idx_val = strtoul(idx, &idx, 0);
4050                 CHECK(!idx[0], EINVAL);
4051
4052                 CHECK(!fsrc->var_size, EINVAL);
4053
4054                 instr->type = INSTR_REGADD_RIM;
4055                 if (src[0] == 'h')
4056                         instr->type = INSTR_REGADD_RIH;
4057
4058                 instr->regarray.regarray_id = r->id;
4059                 instr->regarray.idx_val = idx_val;
4060                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4061                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4062                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4063                 return 0;
4064         }
4065
4066         /* REGADD_RII. */
4067         src_val = strtoull(src, &src, 0);
4068         CHECK(!src[0], EINVAL);
4069
4070         idx_val = strtoul(idx, &idx, 0);
4071         CHECK(!idx[0], EINVAL);
4072
4073         instr->type = INSTR_REGADD_RII;
4074         instr->regarray.idx_val = idx_val;
4075         instr->regarray.dstsrc_val = src_val;
4076         return 0;
4077 }
4078
4079 static inline void
4080 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4081 {
4082         struct thread *t = &p->threads[p->thread_id];
4083         struct instruction *ip = t->ip;
4084
4085         /* Structs. */
4086         __instr_regprefetch_rh_exec(p, t, ip);
4087
4088         /* Thread. */
4089         thread_ip_inc(p);
4090 }
4091
4092 static inline void
4093 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4094 {
4095         struct thread *t = &p->threads[p->thread_id];
4096         struct instruction *ip = t->ip;
4097
4098         /* Structs. */
4099         __instr_regprefetch_rm_exec(p, t, ip);
4100
4101         /* Thread. */
4102         thread_ip_inc(p);
4103 }
4104
4105 static inline void
4106 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4107 {
4108         struct thread *t = &p->threads[p->thread_id];
4109         struct instruction *ip = t->ip;
4110
4111         /* Structs. */
4112         __instr_regprefetch_ri_exec(p, t, ip);
4113
4114         /* Thread. */
4115         thread_ip_inc(p);
4116 }
4117
4118 static inline void
4119 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4120 {
4121         struct thread *t = &p->threads[p->thread_id];
4122         struct instruction *ip = t->ip;
4123
4124         /* Structs. */
4125         __instr_regrd_hrh_exec(p, t, ip);
4126
4127         /* Thread. */
4128         thread_ip_inc(p);
4129 }
4130
4131 static inline void
4132 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4133 {
4134         struct thread *t = &p->threads[p->thread_id];
4135         struct instruction *ip = t->ip;
4136
4137         /* Structs. */
4138         __instr_regrd_hrm_exec(p, t, ip);
4139
4140         /* Thread. */
4141         thread_ip_inc(p);
4142 }
4143
4144 static inline void
4145 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4146 {
4147         struct thread *t = &p->threads[p->thread_id];
4148         struct instruction *ip = t->ip;
4149
4150         /* Structs. */
4151         __instr_regrd_mrh_exec(p, t, ip);
4152
4153         /* Thread. */
4154         thread_ip_inc(p);
4155 }
4156
4157 static inline void
4158 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4159 {
4160         struct thread *t = &p->threads[p->thread_id];
4161         struct instruction *ip = t->ip;
4162
4163         /* Structs. */
4164         __instr_regrd_mrm_exec(p, t, ip);
4165
4166         /* Thread. */
4167         thread_ip_inc(p);
4168 }
4169
4170 static inline void
4171 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4172 {
4173         struct thread *t = &p->threads[p->thread_id];
4174         struct instruction *ip = t->ip;
4175
4176         /* Structs. */
4177         __instr_regrd_hri_exec(p, t, ip);
4178
4179         /* Thread. */
4180         thread_ip_inc(p);
4181 }
4182
4183 static inline void
4184 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4185 {
4186         struct thread *t = &p->threads[p->thread_id];
4187         struct instruction *ip = t->ip;
4188
4189         /* Structs. */
4190         __instr_regrd_mri_exec(p, t, ip);
4191
4192         /* Thread. */
4193         thread_ip_inc(p);
4194 }
4195
4196 static inline void
4197 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4198 {
4199         struct thread *t = &p->threads[p->thread_id];
4200         struct instruction *ip = t->ip;
4201
4202         /* Structs. */
4203         __instr_regwr_rhh_exec(p, t, ip);
4204
4205         /* Thread. */
4206         thread_ip_inc(p);
4207 }
4208
4209 static inline void
4210 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4211 {
4212         struct thread *t = &p->threads[p->thread_id];
4213         struct instruction *ip = t->ip;
4214
4215         /* Structs. */
4216         __instr_regwr_rhm_exec(p, t, ip);
4217
4218         /* Thread. */
4219         thread_ip_inc(p);
4220 }
4221
4222 static inline void
4223 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4224 {
4225         struct thread *t = &p->threads[p->thread_id];
4226         struct instruction *ip = t->ip;
4227
4228         /* Structs. */
4229         __instr_regwr_rmh_exec(p, t, ip);
4230
4231         /* Thread. */
4232         thread_ip_inc(p);
4233 }
4234
4235 static inline void
4236 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4237 {
4238         struct thread *t = &p->threads[p->thread_id];
4239         struct instruction *ip = t->ip;
4240
4241         /* Structs. */
4242         __instr_regwr_rmm_exec(p, t, ip);
4243
4244         /* Thread. */
4245         thread_ip_inc(p);
4246 }
4247
4248 static inline void
4249 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4250 {
4251         struct thread *t = &p->threads[p->thread_id];
4252         struct instruction *ip = t->ip;
4253
4254         /* Structs. */
4255         __instr_regwr_rhi_exec(p, t, ip);
4256
4257         /* Thread. */
4258         thread_ip_inc(p);
4259 }
4260
4261 static inline void
4262 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4263 {
4264         struct thread *t = &p->threads[p->thread_id];
4265         struct instruction *ip = t->ip;
4266
4267         /* Structs. */
4268         __instr_regwr_rmi_exec(p, t, ip);
4269
4270         /* Thread. */
4271         thread_ip_inc(p);
4272 }
4273
4274 static inline void
4275 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4276 {
4277         struct thread *t = &p->threads[p->thread_id];
4278         struct instruction *ip = t->ip;
4279
4280         /* Structs. */
4281         __instr_regwr_rih_exec(p, t, ip);
4282
4283         /* Thread. */
4284         thread_ip_inc(p);
4285 }
4286
4287 static inline void
4288 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4289 {
4290         struct thread *t = &p->threads[p->thread_id];
4291         struct instruction *ip = t->ip;
4292
4293         /* Structs. */
4294         __instr_regwr_rim_exec(p, t, ip);
4295
4296         /* Thread. */
4297         thread_ip_inc(p);
4298 }
4299
4300 static inline void
4301 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4302 {
4303         struct thread *t = &p->threads[p->thread_id];
4304         struct instruction *ip = t->ip;
4305
4306         /* Structs. */
4307         __instr_regwr_rii_exec(p, t, ip);
4308
4309         /* Thread. */
4310         thread_ip_inc(p);
4311 }
4312
4313 static inline void
4314 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4315 {
4316         struct thread *t = &p->threads[p->thread_id];
4317         struct instruction *ip = t->ip;
4318
4319         /* Structs. */
4320         __instr_regadd_rhh_exec(p, t, ip);
4321
4322         /* Thread. */
4323         thread_ip_inc(p);
4324 }
4325
4326 static inline void
4327 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4328 {
4329         struct thread *t = &p->threads[p->thread_id];
4330         struct instruction *ip = t->ip;
4331
4332         /* Structs. */
4333         __instr_regadd_rhm_exec(p, t, ip);
4334
4335         /* Thread. */
4336         thread_ip_inc(p);
4337 }
4338
4339 static inline void
4340 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4341 {
4342         struct thread *t = &p->threads[p->thread_id];
4343         struct instruction *ip = t->ip;
4344
4345         /* Structs. */
4346         __instr_regadd_rmh_exec(p, t, ip);
4347
4348         /* Thread. */
4349         thread_ip_inc(p);
4350 }
4351
4352 static inline void
4353 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4354 {
4355         struct thread *t = &p->threads[p->thread_id];
4356         struct instruction *ip = t->ip;
4357
4358         /* Structs. */
4359         __instr_regadd_rmm_exec(p, t, ip);
4360
4361         /* Thread. */
4362         thread_ip_inc(p);
4363 }
4364
4365 static inline void
4366 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4367 {
4368         struct thread *t = &p->threads[p->thread_id];
4369         struct instruction *ip = t->ip;
4370
4371         /* Structs. */
4372         __instr_regadd_rhi_exec(p, t, ip);
4373
4374         /* Thread. */
4375         thread_ip_inc(p);
4376 }
4377
4378 static inline void
4379 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4380 {
4381         struct thread *t = &p->threads[p->thread_id];
4382         struct instruction *ip = t->ip;
4383
4384         /* Structs. */
4385         __instr_regadd_rmi_exec(p, t, ip);
4386
4387         /* Thread. */
4388         thread_ip_inc(p);
4389 }
4390
4391 static inline void
4392 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4393 {
4394         struct thread *t = &p->threads[p->thread_id];
4395         struct instruction *ip = t->ip;
4396
4397         /* Structs. */
4398         __instr_regadd_rih_exec(p, t, ip);
4399
4400         /* Thread. */
4401         thread_ip_inc(p);
4402 }
4403
4404 static inline void
4405 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4406 {
4407         struct thread *t = &p->threads[p->thread_id];
4408         struct instruction *ip = t->ip;
4409
4410         /* Structs. */
4411         __instr_regadd_rim_exec(p, t, ip);
4412
4413         /* Thread. */
4414         thread_ip_inc(p);
4415 }
4416
4417 static inline void
4418 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4419 {
4420         struct thread *t = &p->threads[p->thread_id];
4421         struct instruction *ip = t->ip;
4422
4423         /* Structs. */
4424         __instr_regadd_rii_exec(p, t, ip);
4425
4426         /* Thread. */
4427         thread_ip_inc(p);
4428 }
4429
4430 /*
4431  * metarray.
4432  */
4433 static struct metarray *
4434 metarray_find(struct rte_swx_pipeline *p, const char *name);
4435
4436 static int
4437 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4438                             struct action *action,
4439                             char **tokens,
4440                             int n_tokens,
4441                             struct instruction *instr,
4442                             struct instruction_data *data __rte_unused)
4443 {
4444         char *metarray = tokens[1], *idx = tokens[2];
4445         struct metarray *m;
4446         struct field *fidx;
4447         uint32_t idx_struct_id, idx_val;
4448
4449         CHECK(n_tokens == 3, EINVAL);
4450
4451         m = metarray_find(p, metarray);
4452         CHECK(m, EINVAL);
4453
4454         /* METPREFETCH_H, METPREFETCH_M. */
4455         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4456         if (fidx) {
4457                 CHECK(!fidx->var_size, EINVAL);
4458
4459                 instr->type = INSTR_METPREFETCH_M;
4460                 if (idx[0] == 'h')
4461                         instr->type = INSTR_METPREFETCH_H;
4462
4463                 instr->meter.metarray_id = m->id;
4464                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4465                 instr->meter.idx.n_bits = fidx->n_bits;
4466                 instr->meter.idx.offset = fidx->offset / 8;
4467                 return 0;
4468         }
4469
4470         /* METPREFETCH_I. */
4471         idx_val = strtoul(idx, &idx, 0);
4472         CHECK(!idx[0], EINVAL);
4473
4474         instr->type = INSTR_METPREFETCH_I;
4475         instr->meter.metarray_id = m->id;
4476         instr->meter.idx_val = idx_val;
4477         return 0;
4478 }
4479
4480 static int
4481 instr_meter_translate(struct rte_swx_pipeline *p,
4482                       struct action *action,
4483                       char **tokens,
4484                       int n_tokens,
4485                       struct instruction *instr,
4486                       struct instruction_data *data __rte_unused)
4487 {
4488         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4489         char *color_in = tokens[4], *color_out = tokens[5];
4490         struct metarray *m;
4491         struct field *fidx, *flength, *fcin, *fcout;
4492         uint32_t idx_struct_id, length_struct_id;
4493         uint32_t color_in_struct_id, color_out_struct_id;
4494
4495         CHECK(n_tokens == 6, EINVAL);
4496
4497         m = metarray_find(p, metarray);
4498         CHECK(m, EINVAL);
4499
4500         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4501
4502         flength = struct_field_parse(p, action, length, &length_struct_id);
4503         CHECK(flength, EINVAL);
4504         CHECK(!flength->var_size, EINVAL);
4505
4506         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4507
4508         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4509         CHECK(fcout, EINVAL);
4510         CHECK(!fcout->var_size, EINVAL);
4511
4512         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4513         if (fidx && fcin) {
4514                 CHECK(!fidx->var_size, EINVAL);
4515                 CHECK(!fcin->var_size, EINVAL);
4516
4517                 instr->type = INSTR_METER_MMM;
4518                 if (idx[0] == 'h' && length[0] == 'h')
4519                         instr->type = INSTR_METER_HHM;
4520                 if (idx[0] == 'h' && length[0] != 'h')
4521                         instr->type = INSTR_METER_HMM;
4522                 if (idx[0] != 'h' && length[0] == 'h')
4523                         instr->type = INSTR_METER_MHM;
4524
4525                 instr->meter.metarray_id = m->id;
4526
4527                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4528                 instr->meter.idx.n_bits = fidx->n_bits;
4529                 instr->meter.idx.offset = fidx->offset / 8;
4530
4531                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4532                 instr->meter.length.n_bits = flength->n_bits;
4533                 instr->meter.length.offset = flength->offset / 8;
4534
4535                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4536                 instr->meter.color_in.n_bits = fcin->n_bits;
4537                 instr->meter.color_in.offset = fcin->offset / 8;
4538
4539                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4540                 instr->meter.color_out.n_bits = fcout->n_bits;
4541                 instr->meter.color_out.offset = fcout->offset / 8;
4542
4543                 return 0;
4544         }
4545
4546         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4547         if (fidx && !fcin) {
4548                 uint32_t color_in_val;
4549
4550                 CHECK(!fidx->var_size, EINVAL);
4551
4552                 color_in_val = strtoul(color_in, &color_in, 0);
4553                 CHECK(!color_in[0], EINVAL);
4554
4555                 instr->type = INSTR_METER_MMI;
4556                 if (idx[0] == 'h' && length[0] == 'h')
4557                         instr->type = INSTR_METER_HHI;
4558                 if (idx[0] == 'h' && length[0] != 'h')
4559                         instr->type = INSTR_METER_HMI;
4560                 if (idx[0] != 'h' && length[0] == 'h')
4561                         instr->type = INSTR_METER_MHI;
4562
4563                 instr->meter.metarray_id = m->id;
4564
4565                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4566                 instr->meter.idx.n_bits = fidx->n_bits;
4567                 instr->meter.idx.offset = fidx->offset / 8;
4568
4569                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4570                 instr->meter.length.n_bits = flength->n_bits;
4571                 instr->meter.length.offset = flength->offset / 8;
4572
4573                 instr->meter.color_in_val = color_in_val;
4574
4575                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4576                 instr->meter.color_out.n_bits = fcout->n_bits;
4577                 instr->meter.color_out.offset = fcout->offset / 8;
4578
4579                 return 0;
4580         }
4581
4582         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4583         if (!fidx && fcin) {
4584                 uint32_t idx_val;
4585
4586                 idx_val = strtoul(idx, &idx, 0);
4587                 CHECK(!idx[0], EINVAL);
4588
4589                 CHECK(!fcin->var_size, EINVAL);
4590
4591                 instr->type = INSTR_METER_IMM;
4592                 if (length[0] == 'h')
4593                         instr->type = INSTR_METER_IHM;
4594
4595                 instr->meter.metarray_id = m->id;
4596
4597                 instr->meter.idx_val = idx_val;
4598
4599                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4600                 instr->meter.length.n_bits = flength->n_bits;
4601                 instr->meter.length.offset = flength->offset / 8;
4602
4603                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4604                 instr->meter.color_in.n_bits = fcin->n_bits;
4605                 instr->meter.color_in.offset = fcin->offset / 8;
4606
4607                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4608                 instr->meter.color_out.n_bits = fcout->n_bits;
4609                 instr->meter.color_out.offset = fcout->offset / 8;
4610
4611                 return 0;
4612         }
4613
4614         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4615         if (!fidx && !fcin) {
4616                 uint32_t idx_val, color_in_val;
4617
4618                 idx_val = strtoul(idx, &idx, 0);
4619                 CHECK(!idx[0], EINVAL);
4620
4621                 color_in_val = strtoul(color_in, &color_in, 0);
4622                 CHECK(!color_in[0], EINVAL);
4623
4624                 instr->type = INSTR_METER_IMI;
4625                 if (length[0] == 'h')
4626                         instr->type = INSTR_METER_IHI;
4627
4628                 instr->meter.metarray_id = m->id;
4629
4630                 instr->meter.idx_val = idx_val;
4631
4632                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4633                 instr->meter.length.n_bits = flength->n_bits;
4634                 instr->meter.length.offset = flength->offset / 8;
4635
4636                 instr->meter.color_in_val = color_in_val;
4637
4638                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4639                 instr->meter.color_out.n_bits = fcout->n_bits;
4640                 instr->meter.color_out.offset = fcout->offset / 8;
4641
4642                 return 0;
4643         }
4644
4645         CHECK(0, EINVAL);
4646 }
4647
4648 static inline void
4649 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4650 {
4651         struct thread *t = &p->threads[p->thread_id];
4652         struct instruction *ip = t->ip;
4653
4654         /* Structs. */
4655         __instr_metprefetch_h_exec(p, t, ip);
4656
4657         /* Thread. */
4658         thread_ip_inc(p);
4659 }
4660
4661 static inline void
4662 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4663 {
4664         struct thread *t = &p->threads[p->thread_id];
4665         struct instruction *ip = t->ip;
4666
4667         /* Structs. */
4668         __instr_metprefetch_m_exec(p, t, ip);
4669
4670         /* Thread. */
4671         thread_ip_inc(p);
4672 }
4673
4674 static inline void
4675 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4676 {
4677         struct thread *t = &p->threads[p->thread_id];
4678         struct instruction *ip = t->ip;
4679
4680         /* Structs. */
4681         __instr_metprefetch_i_exec(p, t, ip);
4682
4683         /* Thread. */
4684         thread_ip_inc(p);
4685 }
4686
4687 static inline void
4688 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4689 {
4690         struct thread *t = &p->threads[p->thread_id];
4691         struct instruction *ip = t->ip;
4692
4693         /* Structs. */
4694         __instr_meter_hhm_exec(p, t, ip);
4695
4696         /* Thread. */
4697         thread_ip_inc(p);
4698 }
4699
4700 static inline void
4701 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4702 {
4703         struct thread *t = &p->threads[p->thread_id];
4704         struct instruction *ip = t->ip;
4705
4706         /* Structs. */
4707         __instr_meter_hhi_exec(p, t, ip);
4708
4709         /* Thread. */
4710         thread_ip_inc(p);
4711 }
4712
4713 static inline void
4714 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4715 {
4716         struct thread *t = &p->threads[p->thread_id];
4717         struct instruction *ip = t->ip;
4718
4719         /* Structs. */
4720         __instr_meter_hmm_exec(p, t, ip);
4721
4722         /* Thread. */
4723         thread_ip_inc(p);
4724 }
4725
4726 static inline void
4727 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4728 {
4729         struct thread *t = &p->threads[p->thread_id];
4730         struct instruction *ip = t->ip;
4731
4732         /* Structs. */
4733         __instr_meter_hmi_exec(p, t, ip);
4734
4735         /* Thread. */
4736         thread_ip_inc(p);
4737 }
4738
4739 static inline void
4740 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4741 {
4742         struct thread *t = &p->threads[p->thread_id];
4743         struct instruction *ip = t->ip;
4744
4745         /* Structs. */
4746         __instr_meter_mhm_exec(p, t, ip);
4747
4748         /* Thread. */
4749         thread_ip_inc(p);
4750 }
4751
4752 static inline void
4753 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4754 {
4755         struct thread *t = &p->threads[p->thread_id];
4756         struct instruction *ip = t->ip;
4757
4758         /* Structs. */
4759         __instr_meter_mhi_exec(p, t, ip);
4760
4761         /* Thread. */
4762         thread_ip_inc(p);
4763 }
4764
4765 static inline void
4766 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4767 {
4768         struct thread *t = &p->threads[p->thread_id];
4769         struct instruction *ip = t->ip;
4770
4771         /* Structs. */
4772         __instr_meter_mmm_exec(p, t, ip);
4773
4774         /* Thread. */
4775         thread_ip_inc(p);
4776 }
4777
4778 static inline void
4779 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4780 {
4781         struct thread *t = &p->threads[p->thread_id];
4782         struct instruction *ip = t->ip;
4783
4784         /* Structs. */
4785         __instr_meter_mmi_exec(p, t, ip);
4786
4787         /* Thread. */
4788         thread_ip_inc(p);
4789 }
4790
4791 static inline void
4792 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4793 {
4794         struct thread *t = &p->threads[p->thread_id];
4795         struct instruction *ip = t->ip;
4796
4797         /* Structs. */
4798         __instr_meter_ihm_exec(p, t, ip);
4799
4800         /* Thread. */
4801         thread_ip_inc(p);
4802 }
4803
4804 static inline void
4805 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4806 {
4807         struct thread *t = &p->threads[p->thread_id];
4808         struct instruction *ip = t->ip;
4809
4810         /* Structs. */
4811         __instr_meter_ihi_exec(p, t, ip);
4812
4813         /* Thread. */
4814         thread_ip_inc(p);
4815 }
4816
4817 static inline void
4818 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4819 {
4820         struct thread *t = &p->threads[p->thread_id];
4821         struct instruction *ip = t->ip;
4822
4823         /* Structs. */
4824         __instr_meter_imm_exec(p, t, ip);
4825
4826         /* Thread. */
4827         thread_ip_inc(p);
4828 }
4829
4830 static inline void
4831 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4832 {
4833         struct thread *t = &p->threads[p->thread_id];
4834         struct instruction *ip = t->ip;
4835
4836         /* Structs. */
4837         __instr_meter_imi_exec(p, t, ip);
4838
4839         /* Thread. */
4840         thread_ip_inc(p);
4841 }
4842
4843 /*
4844  * jmp.
4845  */
4846 static int
4847 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4848                     struct action *action __rte_unused,
4849                     char **tokens,
4850                     int n_tokens,
4851                     struct instruction *instr,
4852                     struct instruction_data *data)
4853 {
4854         CHECK(n_tokens == 2, EINVAL);
4855
4856         strcpy(data->jmp_label, tokens[1]);
4857
4858         instr->type = INSTR_JMP;
4859         instr->jmp.ip = NULL; /* Resolved later. */
4860         return 0;
4861 }
4862
4863 static int
4864 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4865                           struct action *action __rte_unused,
4866                           char **tokens,
4867                           int n_tokens,
4868                           struct instruction *instr,
4869                           struct instruction_data *data)
4870 {
4871         struct header *h;
4872
4873         CHECK(n_tokens == 3, EINVAL);
4874
4875         strcpy(data->jmp_label, tokens[1]);
4876
4877         h = header_parse(p, tokens[2]);
4878         CHECK(h, EINVAL);
4879
4880         instr->type = INSTR_JMP_VALID;
4881         instr->jmp.ip = NULL; /* Resolved later. */
4882         instr->jmp.header_id = h->id;
4883         return 0;
4884 }
4885
4886 static int
4887 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4888                             struct action *action __rte_unused,
4889                             char **tokens,
4890                             int n_tokens,
4891                             struct instruction *instr,
4892                             struct instruction_data *data)
4893 {
4894         struct header *h;
4895
4896         CHECK(n_tokens == 3, EINVAL);
4897
4898         strcpy(data->jmp_label, tokens[1]);
4899
4900         h = header_parse(p, tokens[2]);
4901         CHECK(h, EINVAL);
4902
4903         instr->type = INSTR_JMP_INVALID;
4904         instr->jmp.ip = NULL; /* Resolved later. */
4905         instr->jmp.header_id = h->id;
4906         return 0;
4907 }
4908
4909 static int
4910 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4911                         struct action *action,
4912                         char **tokens,
4913                         int n_tokens,
4914                         struct instruction *instr,
4915                         struct instruction_data *data)
4916 {
4917         CHECK(!action, EINVAL);
4918         CHECK(n_tokens == 2, EINVAL);
4919
4920         strcpy(data->jmp_label, tokens[1]);
4921
4922         instr->type = INSTR_JMP_HIT;
4923         instr->jmp.ip = NULL; /* Resolved later. */
4924         return 0;
4925 }
4926
4927 static int
4928 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4929                          struct action *action,
4930                          char **tokens,
4931                          int n_tokens,
4932                          struct instruction *instr,
4933                          struct instruction_data *data)
4934 {
4935         CHECK(!action, EINVAL);
4936         CHECK(n_tokens == 2, EINVAL);
4937
4938         strcpy(data->jmp_label, tokens[1]);
4939
4940         instr->type = INSTR_JMP_MISS;
4941         instr->jmp.ip = NULL; /* Resolved later. */
4942         return 0;
4943 }
4944
4945 static int
4946 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4947                                struct action *action,
4948                                char **tokens,
4949                                int n_tokens,
4950                                struct instruction *instr,
4951                                struct instruction_data *data)
4952 {
4953         struct action *a;
4954
4955         CHECK(!action, EINVAL);
4956         CHECK(n_tokens == 3, EINVAL);
4957
4958         strcpy(data->jmp_label, tokens[1]);
4959
4960         a = action_find(p, tokens[2]);
4961         CHECK(a, EINVAL);
4962
4963         instr->type = INSTR_JMP_ACTION_HIT;
4964         instr->jmp.ip = NULL; /* Resolved later. */
4965         instr->jmp.action_id = a->id;
4966         return 0;
4967 }
4968
4969 static int
4970 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4971                                 struct action *action,
4972                                 char **tokens,
4973                                 int n_tokens,
4974                                 struct instruction *instr,
4975                                 struct instruction_data *data)
4976 {
4977         struct action *a;
4978
4979         CHECK(!action, EINVAL);
4980         CHECK(n_tokens == 3, EINVAL);
4981
4982         strcpy(data->jmp_label, tokens[1]);
4983
4984         a = action_find(p, tokens[2]);
4985         CHECK(a, EINVAL);
4986
4987         instr->type = INSTR_JMP_ACTION_MISS;
4988         instr->jmp.ip = NULL; /* Resolved later. */
4989         instr->jmp.action_id = a->id;
4990         return 0;
4991 }
4992
4993 static int
4994 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4995                        struct action *action,
4996                        char **tokens,
4997                        int n_tokens,
4998                        struct instruction *instr,
4999                        struct instruction_data *data)
5000 {
5001         char *a = tokens[2], *b = tokens[3];
5002         struct field *fa, *fb;
5003         uint64_t b_val;
5004         uint32_t a_struct_id, b_struct_id;
5005
5006         CHECK(n_tokens == 4, EINVAL);
5007
5008         strcpy(data->jmp_label, tokens[1]);
5009
5010         fa = struct_field_parse(p, action, a, &a_struct_id);
5011         CHECK(fa, EINVAL);
5012         CHECK(!fa->var_size, EINVAL);
5013
5014         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5015         fb = struct_field_parse(p, action, b, &b_struct_id);
5016         if (fb) {
5017                 CHECK(!fb->var_size, EINVAL);
5018
5019                 instr->type = INSTR_JMP_EQ;
5020                 if (a[0] != 'h' && b[0] == 'h')
5021                         instr->type = INSTR_JMP_EQ_MH;
5022                 if (a[0] == 'h' && b[0] != 'h')
5023                         instr->type = INSTR_JMP_EQ_HM;
5024                 if (a[0] == 'h' && b[0] == 'h')
5025                         instr->type = INSTR_JMP_EQ_HH;
5026                 instr->jmp.ip = NULL; /* Resolved later. */
5027
5028                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5029                 instr->jmp.a.n_bits = fa->n_bits;
5030                 instr->jmp.a.offset = fa->offset / 8;
5031                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5032                 instr->jmp.b.n_bits = fb->n_bits;
5033                 instr->jmp.b.offset = fb->offset / 8;
5034                 return 0;
5035         }
5036
5037         /* JMP_EQ_I. */
5038         b_val = strtoull(b, &b, 0);
5039         CHECK(!b[0], EINVAL);
5040
5041         if (a[0] == 'h')
5042                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5043
5044         instr->type = INSTR_JMP_EQ_I;
5045         instr->jmp.ip = NULL; /* Resolved later. */
5046         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5047         instr->jmp.a.n_bits = fa->n_bits;
5048         instr->jmp.a.offset = fa->offset / 8;
5049         instr->jmp.b_val = b_val;
5050         return 0;
5051 }
5052
5053 static int
5054 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5055                         struct action *action,
5056                         char **tokens,
5057                         int n_tokens,
5058                         struct instruction *instr,
5059                         struct instruction_data *data)
5060 {
5061         char *a = tokens[2], *b = tokens[3];
5062         struct field *fa, *fb;
5063         uint64_t b_val;
5064         uint32_t a_struct_id, b_struct_id;
5065
5066         CHECK(n_tokens == 4, EINVAL);
5067
5068         strcpy(data->jmp_label, tokens[1]);
5069
5070         fa = struct_field_parse(p, action, a, &a_struct_id);
5071         CHECK(fa, EINVAL);
5072         CHECK(!fa->var_size, EINVAL);
5073
5074         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5075         fb = struct_field_parse(p, action, b, &b_struct_id);
5076         if (fb) {
5077                 CHECK(!fb->var_size, EINVAL);
5078
5079                 instr->type = INSTR_JMP_NEQ;
5080                 if (a[0] != 'h' && b[0] == 'h')
5081                         instr->type = INSTR_JMP_NEQ_MH;
5082                 if (a[0] == 'h' && b[0] != 'h')
5083                         instr->type = INSTR_JMP_NEQ_HM;
5084                 if (a[0] == 'h' && b[0] == 'h')
5085                         instr->type = INSTR_JMP_NEQ_HH;
5086                 instr->jmp.ip = NULL; /* Resolved later. */
5087
5088                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5089                 instr->jmp.a.n_bits = fa->n_bits;
5090                 instr->jmp.a.offset = fa->offset / 8;
5091                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5092                 instr->jmp.b.n_bits = fb->n_bits;
5093                 instr->jmp.b.offset = fb->offset / 8;
5094                 return 0;
5095         }
5096
5097         /* JMP_NEQ_I. */
5098         b_val = strtoull(b, &b, 0);
5099         CHECK(!b[0], EINVAL);
5100
5101         if (a[0] == 'h')
5102                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5103
5104         instr->type = INSTR_JMP_NEQ_I;
5105         instr->jmp.ip = NULL; /* Resolved later. */
5106         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5107         instr->jmp.a.n_bits = fa->n_bits;
5108         instr->jmp.a.offset = fa->offset / 8;
5109         instr->jmp.b_val = b_val;
5110         return 0;
5111 }
5112
5113 static int
5114 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5115                        struct action *action,
5116                        char **tokens,
5117                        int n_tokens,
5118                        struct instruction *instr,
5119                        struct instruction_data *data)
5120 {
5121         char *a = tokens[2], *b = tokens[3];
5122         struct field *fa, *fb;
5123         uint64_t b_val;
5124         uint32_t a_struct_id, b_struct_id;
5125
5126         CHECK(n_tokens == 4, EINVAL);
5127
5128         strcpy(data->jmp_label, tokens[1]);
5129
5130         fa = struct_field_parse(p, action, a, &a_struct_id);
5131         CHECK(fa, EINVAL);
5132         CHECK(!fa->var_size, EINVAL);
5133
5134         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5135         fb = struct_field_parse(p, action, b, &b_struct_id);
5136         if (fb) {
5137                 CHECK(!fb->var_size, EINVAL);
5138
5139                 instr->type = INSTR_JMP_LT;
5140                 if (a[0] == 'h' && b[0] != 'h')
5141                         instr->type = INSTR_JMP_LT_HM;
5142                 if (a[0] != 'h' && b[0] == 'h')
5143                         instr->type = INSTR_JMP_LT_MH;
5144                 if (a[0] == 'h' && b[0] == 'h')
5145                         instr->type = INSTR_JMP_LT_HH;
5146                 instr->jmp.ip = NULL; /* Resolved later. */
5147
5148                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5149                 instr->jmp.a.n_bits = fa->n_bits;
5150                 instr->jmp.a.offset = fa->offset / 8;
5151                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5152                 instr->jmp.b.n_bits = fb->n_bits;
5153                 instr->jmp.b.offset = fb->offset / 8;
5154                 return 0;
5155         }
5156
5157         /* JMP_LT_MI, JMP_LT_HI. */
5158         b_val = strtoull(b, &b, 0);
5159         CHECK(!b[0], EINVAL);
5160
5161         instr->type = INSTR_JMP_LT_MI;
5162         if (a[0] == 'h')
5163                 instr->type = INSTR_JMP_LT_HI;
5164         instr->jmp.ip = NULL; /* Resolved later. */
5165
5166         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5167         instr->jmp.a.n_bits = fa->n_bits;
5168         instr->jmp.a.offset = fa->offset / 8;
5169         instr->jmp.b_val = b_val;
5170         return 0;
5171 }
5172
5173 static int
5174 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5175                        struct action *action,
5176                        char **tokens,
5177                        int n_tokens,
5178                        struct instruction *instr,
5179                        struct instruction_data *data)
5180 {
5181         char *a = tokens[2], *b = tokens[3];
5182         struct field *fa, *fb;
5183         uint64_t b_val;
5184         uint32_t a_struct_id, b_struct_id;
5185
5186         CHECK(n_tokens == 4, EINVAL);
5187
5188         strcpy(data->jmp_label, tokens[1]);
5189
5190         fa = struct_field_parse(p, action, a, &a_struct_id);
5191         CHECK(fa, EINVAL);
5192         CHECK(!fa->var_size, EINVAL);
5193
5194         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5195         fb = struct_field_parse(p, action, b, &b_struct_id);
5196         if (fb) {
5197                 CHECK(!fb->var_size, EINVAL);
5198
5199                 instr->type = INSTR_JMP_GT;
5200                 if (a[0] == 'h' && b[0] != 'h')
5201                         instr->type = INSTR_JMP_GT_HM;
5202                 if (a[0] != 'h' && b[0] == 'h')
5203                         instr->type = INSTR_JMP_GT_MH;
5204                 if (a[0] == 'h' && b[0] == 'h')
5205                         instr->type = INSTR_JMP_GT_HH;
5206                 instr->jmp.ip = NULL; /* Resolved later. */
5207
5208                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5209                 instr->jmp.a.n_bits = fa->n_bits;
5210                 instr->jmp.a.offset = fa->offset / 8;
5211                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5212                 instr->jmp.b.n_bits = fb->n_bits;
5213                 instr->jmp.b.offset = fb->offset / 8;
5214                 return 0;
5215         }
5216
5217         /* JMP_GT_MI, JMP_GT_HI. */
5218         b_val = strtoull(b, &b, 0);
5219         CHECK(!b[0], EINVAL);
5220
5221         instr->type = INSTR_JMP_GT_MI;
5222         if (a[0] == 'h')
5223                 instr->type = INSTR_JMP_GT_HI;
5224         instr->jmp.ip = NULL; /* Resolved later. */
5225
5226         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5227         instr->jmp.a.n_bits = fa->n_bits;
5228         instr->jmp.a.offset = fa->offset / 8;
5229         instr->jmp.b_val = b_val;
5230         return 0;
5231 }
5232
5233 static inline void
5234 instr_jmp_exec(struct rte_swx_pipeline *p)
5235 {
5236         struct thread *t = &p->threads[p->thread_id];
5237         struct instruction *ip = t->ip;
5238
5239         TRACE("[Thread %2u] jmp\n", p->thread_id);
5240
5241         thread_ip_set(t, ip->jmp.ip);
5242 }
5243
5244 static inline void
5245 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5246 {
5247         struct thread *t = &p->threads[p->thread_id];
5248         struct instruction *ip = t->ip;
5249         uint32_t header_id = ip->jmp.header_id;
5250
5251         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5252
5253         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5254 }
5255
5256 static inline void
5257 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5258 {
5259         struct thread *t = &p->threads[p->thread_id];
5260         struct instruction *ip = t->ip;
5261         uint32_t header_id = ip->jmp.header_id;
5262
5263         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5264
5265         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5266 }
5267
5268 static inline void
5269 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5270 {
5271         struct thread *t = &p->threads[p->thread_id];
5272         struct instruction *ip = t->ip;
5273         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5274
5275         TRACE("[Thread %2u] jmph\n", p->thread_id);
5276
5277         t->ip = ip_next[t->hit];
5278 }
5279
5280 static inline void
5281 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5282 {
5283         struct thread *t = &p->threads[p->thread_id];
5284         struct instruction *ip = t->ip;
5285         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5286
5287         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5288
5289         t->ip = ip_next[t->hit];
5290 }
5291
5292 static inline void
5293 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5294 {
5295         struct thread *t = &p->threads[p->thread_id];
5296         struct instruction *ip = t->ip;
5297
5298         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5299
5300         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5301 }
5302
5303 static inline void
5304 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5305 {
5306         struct thread *t = &p->threads[p->thread_id];
5307         struct instruction *ip = t->ip;
5308
5309         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5310
5311         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5312 }
5313
5314 static inline void
5315 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5316 {
5317         struct thread *t = &p->threads[p->thread_id];
5318         struct instruction *ip = t->ip;
5319
5320         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5321
5322         JMP_CMP(t, ip, ==);
5323 }
5324
5325 static inline void
5326 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5327 {
5328         struct thread *t = &p->threads[p->thread_id];
5329         struct instruction *ip = t->ip;
5330
5331         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5332
5333         JMP_CMP_MH(t, ip, ==);
5334 }
5335
5336 static inline void
5337 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5338 {
5339         struct thread *t = &p->threads[p->thread_id];
5340         struct instruction *ip = t->ip;
5341
5342         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5343
5344         JMP_CMP_HM(t, ip, ==);
5345 }
5346
5347 static inline void
5348 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5349 {
5350         struct thread *t = &p->threads[p->thread_id];
5351         struct instruction *ip = t->ip;
5352
5353         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5354
5355         JMP_CMP_HH_FAST(t, ip, ==);
5356 }
5357
5358 static inline void
5359 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5360 {
5361         struct thread *t = &p->threads[p->thread_id];
5362         struct instruction *ip = t->ip;
5363
5364         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5365
5366         JMP_CMP_I(t, ip, ==);
5367 }
5368
5369 static inline void
5370 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5371 {
5372         struct thread *t = &p->threads[p->thread_id];
5373         struct instruction *ip = t->ip;
5374
5375         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5376
5377         JMP_CMP(t, ip, !=);
5378 }
5379
5380 static inline void
5381 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5382 {
5383         struct thread *t = &p->threads[p->thread_id];
5384         struct instruction *ip = t->ip;
5385
5386         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5387
5388         JMP_CMP_MH(t, ip, !=);
5389 }
5390
5391 static inline void
5392 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5393 {
5394         struct thread *t = &p->threads[p->thread_id];
5395         struct instruction *ip = t->ip;
5396
5397         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5398
5399         JMP_CMP_HM(t, ip, !=);
5400 }
5401
5402 static inline void
5403 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5404 {
5405         struct thread *t = &p->threads[p->thread_id];
5406         struct instruction *ip = t->ip;
5407
5408         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5409
5410         JMP_CMP_HH_FAST(t, ip, !=);
5411 }
5412
5413 static inline void
5414 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5415 {
5416         struct thread *t = &p->threads[p->thread_id];
5417         struct instruction *ip = t->ip;
5418
5419         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5420
5421         JMP_CMP_I(t, ip, !=);
5422 }
5423
5424 static inline void
5425 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5426 {
5427         struct thread *t = &p->threads[p->thread_id];
5428         struct instruction *ip = t->ip;
5429
5430         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5431
5432         JMP_CMP(t, ip, <);
5433 }
5434
5435 static inline void
5436 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5437 {
5438         struct thread *t = &p->threads[p->thread_id];
5439         struct instruction *ip = t->ip;
5440
5441         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5442
5443         JMP_CMP_MH(t, ip, <);
5444 }
5445
5446 static inline void
5447 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5448 {
5449         struct thread *t = &p->threads[p->thread_id];
5450         struct instruction *ip = t->ip;
5451
5452         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5453
5454         JMP_CMP_HM(t, ip, <);
5455 }
5456
5457 static inline void
5458 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5459 {
5460         struct thread *t = &p->threads[p->thread_id];
5461         struct instruction *ip = t->ip;
5462
5463         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5464
5465         JMP_CMP_HH(t, ip, <);
5466 }
5467
5468 static inline void
5469 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5470 {
5471         struct thread *t = &p->threads[p->thread_id];
5472         struct instruction *ip = t->ip;
5473
5474         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5475
5476         JMP_CMP_MI(t, ip, <);
5477 }
5478
5479 static inline void
5480 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5481 {
5482         struct thread *t = &p->threads[p->thread_id];
5483         struct instruction *ip = t->ip;
5484
5485         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5486
5487         JMP_CMP_HI(t, ip, <);
5488 }
5489
5490 static inline void
5491 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5492 {
5493         struct thread *t = &p->threads[p->thread_id];
5494         struct instruction *ip = t->ip;
5495
5496         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5497
5498         JMP_CMP(t, ip, >);
5499 }
5500
5501 static inline void
5502 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5503 {
5504         struct thread *t = &p->threads[p->thread_id];
5505         struct instruction *ip = t->ip;
5506
5507         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5508
5509         JMP_CMP_MH(t, ip, >);
5510 }
5511
5512 static inline void
5513 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5514 {
5515         struct thread *t = &p->threads[p->thread_id];
5516         struct instruction *ip = t->ip;
5517
5518         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5519
5520         JMP_CMP_HM(t, ip, >);
5521 }
5522
5523 static inline void
5524 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5525 {
5526         struct thread *t = &p->threads[p->thread_id];
5527         struct instruction *ip = t->ip;
5528
5529         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5530
5531         JMP_CMP_HH(t, ip, >);
5532 }
5533
5534 static inline void
5535 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5536 {
5537         struct thread *t = &p->threads[p->thread_id];
5538         struct instruction *ip = t->ip;
5539
5540         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5541
5542         JMP_CMP_MI(t, ip, >);
5543 }
5544
5545 static inline void
5546 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5547 {
5548         struct thread *t = &p->threads[p->thread_id];
5549         struct instruction *ip = t->ip;
5550
5551         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5552
5553         JMP_CMP_HI(t, ip, >);
5554 }
5555
5556 /*
5557  * return.
5558  */
5559 static int
5560 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5561                        struct action *action,
5562                        char **tokens __rte_unused,
5563                        int n_tokens,
5564                        struct instruction *instr,
5565                        struct instruction_data *data __rte_unused)
5566 {
5567         CHECK(action, EINVAL);
5568         CHECK(n_tokens == 1, EINVAL);
5569
5570         instr->type = INSTR_RETURN;
5571         return 0;
5572 }
5573
5574 static inline void
5575 instr_return_exec(struct rte_swx_pipeline *p)
5576 {
5577         struct thread *t = &p->threads[p->thread_id];
5578
5579         TRACE("[Thread %2u] return\n", p->thread_id);
5580
5581         t->ip = t->ret;
5582 }
5583
5584 static int
5585 instr_translate(struct rte_swx_pipeline *p,
5586                 struct action *action,
5587                 char *string,
5588                 struct instruction *instr,
5589                 struct instruction_data *data)
5590 {
5591         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5592         int n_tokens = 0, tpos = 0;
5593
5594         /* Parse the instruction string into tokens. */
5595         for ( ; ; ) {
5596                 char *token;
5597
5598                 token = strtok_r(string, " \t\v", &string);
5599                 if (!token)
5600                         break;
5601
5602                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5603                 CHECK_NAME(token, EINVAL);
5604
5605                 tokens[n_tokens] = token;
5606                 n_tokens++;
5607         }
5608
5609         CHECK(n_tokens, EINVAL);
5610
5611         /* Handle the optional instruction label. */
5612         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5613                 strcpy(data->label, tokens[0]);
5614
5615                 tpos += 2;
5616                 CHECK(n_tokens - tpos, EINVAL);
5617         }
5618
5619         /* Identify the instruction type. */
5620         if (!strcmp(tokens[tpos], "rx"))
5621                 return instr_rx_translate(p,
5622                                           action,
5623                                           &tokens[tpos],
5624                                           n_tokens - tpos,
5625                                           instr,
5626                                           data);
5627
5628         if (!strcmp(tokens[tpos], "tx"))
5629                 return instr_tx_translate(p,
5630                                           action,
5631                                           &tokens[tpos],
5632                                           n_tokens - tpos,
5633                                           instr,
5634                                           data);
5635
5636         if (!strcmp(tokens[tpos], "drop"))
5637                 return instr_drop_translate(p,
5638                                             action,
5639                                             &tokens[tpos],
5640                                             n_tokens - tpos,
5641                                             instr,
5642                                             data);
5643
5644         if (!strcmp(tokens[tpos], "extract"))
5645                 return instr_hdr_extract_translate(p,
5646                                                    action,
5647                                                    &tokens[tpos],
5648                                                    n_tokens - tpos,
5649                                                    instr,
5650                                                    data);
5651
5652         if (!strcmp(tokens[tpos], "lookahead"))
5653                 return instr_hdr_lookahead_translate(p,
5654                                                      action,
5655                                                      &tokens[tpos],
5656                                                      n_tokens - tpos,
5657                                                      instr,
5658                                                      data);
5659
5660         if (!strcmp(tokens[tpos], "emit"))
5661                 return instr_hdr_emit_translate(p,
5662                                                 action,
5663                                                 &tokens[tpos],
5664                                                 n_tokens - tpos,
5665                                                 instr,
5666                                                 data);
5667
5668         if (!strcmp(tokens[tpos], "validate"))
5669                 return instr_hdr_validate_translate(p,
5670                                                     action,
5671                                                     &tokens[tpos],
5672                                                     n_tokens - tpos,
5673                                                     instr,
5674                                                     data);
5675
5676         if (!strcmp(tokens[tpos], "invalidate"))
5677                 return instr_hdr_invalidate_translate(p,
5678                                                       action,
5679                                                       &tokens[tpos],
5680                                                       n_tokens - tpos,
5681                                                       instr,
5682                                                       data);
5683
5684         if (!strcmp(tokens[tpos], "mov"))
5685                 return instr_mov_translate(p,
5686                                            action,
5687                                            &tokens[tpos],
5688                                            n_tokens - tpos,
5689                                            instr,
5690                                            data);
5691
5692         if (!strcmp(tokens[tpos], "add"))
5693                 return instr_alu_add_translate(p,
5694                                                action,
5695                                                &tokens[tpos],
5696                                                n_tokens - tpos,
5697                                                instr,
5698                                                data);
5699
5700         if (!strcmp(tokens[tpos], "sub"))
5701                 return instr_alu_sub_translate(p,
5702                                                action,
5703                                                &tokens[tpos],
5704                                                n_tokens - tpos,
5705                                                instr,
5706                                                data);
5707
5708         if (!strcmp(tokens[tpos], "ckadd"))
5709                 return instr_alu_ckadd_translate(p,
5710                                                  action,
5711                                                  &tokens[tpos],
5712                                                  n_tokens - tpos,
5713                                                  instr,
5714                                                  data);
5715
5716         if (!strcmp(tokens[tpos], "cksub"))
5717                 return instr_alu_cksub_translate(p,
5718                                                  action,
5719                                                  &tokens[tpos],
5720                                                  n_tokens - tpos,
5721                                                  instr,
5722                                                  data);
5723
5724         if (!strcmp(tokens[tpos], "and"))
5725                 return instr_alu_and_translate(p,
5726                                                action,
5727                                                &tokens[tpos],
5728                                                n_tokens - tpos,
5729                                                instr,
5730                                                data);
5731
5732         if (!strcmp(tokens[tpos], "or"))
5733                 return instr_alu_or_translate(p,
5734                                               action,
5735                                               &tokens[tpos],
5736                                               n_tokens - tpos,
5737                                               instr,
5738                                               data);
5739
5740         if (!strcmp(tokens[tpos], "xor"))
5741                 return instr_alu_xor_translate(p,
5742                                                action,
5743                                                &tokens[tpos],
5744                                                n_tokens - tpos,
5745                                                instr,
5746                                                data);
5747
5748         if (!strcmp(tokens[tpos], "shl"))
5749                 return instr_alu_shl_translate(p,
5750                                                action,
5751                                                &tokens[tpos],
5752                                                n_tokens - tpos,
5753                                                instr,
5754                                                data);
5755
5756         if (!strcmp(tokens[tpos], "shr"))
5757                 return instr_alu_shr_translate(p,
5758                                                action,
5759                                                &tokens[tpos],
5760                                                n_tokens - tpos,
5761                                                instr,
5762                                                data);
5763
5764         if (!strcmp(tokens[tpos], "regprefetch"))
5765                 return instr_regprefetch_translate(p,
5766                                                    action,
5767                                                    &tokens[tpos],
5768                                                    n_tokens - tpos,
5769                                                    instr,
5770                                                    data);
5771
5772         if (!strcmp(tokens[tpos], "regrd"))
5773                 return instr_regrd_translate(p,
5774                                              action,
5775                                              &tokens[tpos],
5776                                              n_tokens - tpos,
5777                                              instr,
5778                                              data);
5779
5780         if (!strcmp(tokens[tpos], "regwr"))
5781                 return instr_regwr_translate(p,
5782                                              action,
5783                                              &tokens[tpos],
5784                                              n_tokens - tpos,
5785                                              instr,
5786                                              data);
5787
5788         if (!strcmp(tokens[tpos], "regadd"))
5789                 return instr_regadd_translate(p,
5790                                               action,
5791                                               &tokens[tpos],
5792                                               n_tokens - tpos,
5793                                               instr,
5794                                               data);
5795
5796         if (!strcmp(tokens[tpos], "metprefetch"))
5797                 return instr_metprefetch_translate(p,
5798                                                    action,
5799                                                    &tokens[tpos],
5800                                                    n_tokens - tpos,
5801                                                    instr,
5802                                                    data);
5803
5804         if (!strcmp(tokens[tpos], "meter"))
5805                 return instr_meter_translate(p,
5806                                              action,
5807                                              &tokens[tpos],
5808                                              n_tokens - tpos,
5809                                              instr,
5810                                              data);
5811
5812         if (!strcmp(tokens[tpos], "table"))
5813                 return instr_table_translate(p,
5814                                              action,
5815                                              &tokens[tpos],
5816                                              n_tokens - tpos,
5817                                              instr,
5818                                              data);
5819
5820         if (!strcmp(tokens[tpos], "learn"))
5821                 return instr_learn_translate(p,
5822                                              action,
5823                                              &tokens[tpos],
5824                                              n_tokens - tpos,
5825                                              instr,
5826                                              data);
5827
5828         if (!strcmp(tokens[tpos], "forget"))
5829                 return instr_forget_translate(p,
5830                                               action,
5831                                               &tokens[tpos],
5832                                               n_tokens - tpos,
5833                                               instr,
5834                                               data);
5835
5836         if (!strcmp(tokens[tpos], "extern"))
5837                 return instr_extern_translate(p,
5838                                               action,
5839                                               &tokens[tpos],
5840                                               n_tokens - tpos,
5841                                               instr,
5842                                               data);
5843
5844         if (!strcmp(tokens[tpos], "jmp"))
5845                 return instr_jmp_translate(p,
5846                                            action,
5847                                            &tokens[tpos],
5848                                            n_tokens - tpos,
5849                                            instr,
5850                                            data);
5851
5852         if (!strcmp(tokens[tpos], "jmpv"))
5853                 return instr_jmp_valid_translate(p,
5854                                                  action,
5855                                                  &tokens[tpos],
5856                                                  n_tokens - tpos,
5857                                                  instr,
5858                                                  data);
5859
5860         if (!strcmp(tokens[tpos], "jmpnv"))
5861                 return instr_jmp_invalid_translate(p,
5862                                                    action,
5863                                                    &tokens[tpos],
5864                                                    n_tokens - tpos,
5865                                                    instr,
5866                                                    data);
5867
5868         if (!strcmp(tokens[tpos], "jmph"))
5869                 return instr_jmp_hit_translate(p,
5870                                                action,
5871                                                &tokens[tpos],
5872                                                n_tokens - tpos,
5873                                                instr,
5874                                                data);
5875
5876         if (!strcmp(tokens[tpos], "jmpnh"))
5877                 return instr_jmp_miss_translate(p,
5878                                                 action,
5879                                                 &tokens[tpos],
5880                                                 n_tokens - tpos,
5881                                                 instr,
5882                                                 data);
5883
5884         if (!strcmp(tokens[tpos], "jmpa"))
5885                 return instr_jmp_action_hit_translate(p,
5886                                                       action,
5887                                                       &tokens[tpos],
5888                                                       n_tokens - tpos,
5889                                                       instr,
5890                                                       data);
5891
5892         if (!strcmp(tokens[tpos], "jmpna"))
5893                 return instr_jmp_action_miss_translate(p,
5894                                                        action,
5895                                                        &tokens[tpos],
5896                                                        n_tokens - tpos,
5897                                                        instr,
5898                                                        data);
5899
5900         if (!strcmp(tokens[tpos], "jmpeq"))
5901                 return instr_jmp_eq_translate(p,
5902                                               action,
5903                                               &tokens[tpos],
5904                                               n_tokens - tpos,
5905                                               instr,
5906                                               data);
5907
5908         if (!strcmp(tokens[tpos], "jmpneq"))
5909                 return instr_jmp_neq_translate(p,
5910                                                action,
5911                                                &tokens[tpos],
5912                                                n_tokens - tpos,
5913                                                instr,
5914                                                data);
5915
5916         if (!strcmp(tokens[tpos], "jmplt"))
5917                 return instr_jmp_lt_translate(p,
5918                                               action,
5919                                               &tokens[tpos],
5920                                               n_tokens - tpos,
5921                                               instr,
5922                                               data);
5923
5924         if (!strcmp(tokens[tpos], "jmpgt"))
5925                 return instr_jmp_gt_translate(p,
5926                                               action,
5927                                               &tokens[tpos],
5928                                               n_tokens - tpos,
5929                                               instr,
5930                                               data);
5931
5932         if (!strcmp(tokens[tpos], "return"))
5933                 return instr_return_translate(p,
5934                                               action,
5935                                               &tokens[tpos],
5936                                               n_tokens - tpos,
5937                                               instr,
5938                                               data);
5939
5940         CHECK(0, EINVAL);
5941 }
5942
5943 static struct instruction_data *
5944 label_find(struct instruction_data *data, uint32_t n, const char *label)
5945 {
5946         uint32_t i;
5947
5948         for (i = 0; i < n; i++)
5949                 if (!strcmp(label, data[i].label))
5950                         return &data[i];
5951
5952         return NULL;
5953 }
5954
5955 static uint32_t
5956 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5957 {
5958         uint32_t count = 0, i;
5959
5960         if (!label[0])
5961                 return 0;
5962
5963         for (i = 0; i < n; i++)
5964                 if (!strcmp(label, data[i].jmp_label))
5965                         count++;
5966
5967         return count;
5968 }
5969
5970 static int
5971 instr_label_check(struct instruction_data *instruction_data,
5972                   uint32_t n_instructions)
5973 {
5974         uint32_t i;
5975
5976         /* Check that all instruction labels are unique. */
5977         for (i = 0; i < n_instructions; i++) {
5978                 struct instruction_data *data = &instruction_data[i];
5979                 char *label = data->label;
5980                 uint32_t j;
5981
5982                 if (!label[0])
5983                         continue;
5984
5985                 for (j = i + 1; j < n_instructions; j++)
5986                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
5987         }
5988
5989         /* Get users for each instruction label. */
5990         for (i = 0; i < n_instructions; i++) {
5991                 struct instruction_data *data = &instruction_data[i];
5992                 char *label = data->label;
5993
5994                 data->n_users = label_is_used(instruction_data,
5995                                               n_instructions,
5996                                               label);
5997         }
5998
5999         return 0;
6000 }
6001
6002 static int
6003 instr_jmp_resolve(struct instruction *instructions,
6004                   struct instruction_data *instruction_data,
6005                   uint32_t n_instructions)
6006 {
6007         uint32_t i;
6008
6009         for (i = 0; i < n_instructions; i++) {
6010                 struct instruction *instr = &instructions[i];
6011                 struct instruction_data *data = &instruction_data[i];
6012                 struct instruction_data *found;
6013
6014                 if (!instruction_is_jmp(instr))
6015                         continue;
6016
6017                 found = label_find(instruction_data,
6018                                    n_instructions,
6019                                    data->jmp_label);
6020                 CHECK(found, EINVAL);
6021
6022                 instr->jmp.ip = &instructions[found - instruction_data];
6023         }
6024
6025         return 0;
6026 }
6027
6028 static int
6029 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6030              struct action *a,
6031              struct instruction *instr,
6032              struct instruction_data *data __rte_unused,
6033              uint32_t n_instructions)
6034 {
6035         if (!a) {
6036                 enum instruction_type type;
6037                 uint32_t i;
6038
6039                 /* Check that the first instruction is rx. */
6040                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6041
6042                 /* Check that there is at least one tx instruction. */
6043                 for (i = 0; i < n_instructions; i++) {
6044                         type = instr[i].type;
6045
6046                         if (instruction_is_tx(type))
6047                                 break;
6048                 }
6049                 CHECK(i < n_instructions, EINVAL);
6050
6051                 /* Check that the last instruction is either tx or unconditional
6052                  * jump.
6053                  */
6054                 type = instr[n_instructions - 1].type;
6055                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6056         }
6057
6058         if (a) {
6059                 enum instruction_type type;
6060                 uint32_t i;
6061
6062                 /* Check that there is at least one return or tx instruction. */
6063                 for (i = 0; i < n_instructions; i++) {
6064                         type = instr[i].type;
6065
6066                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6067                                 break;
6068                 }
6069                 CHECK(i < n_instructions, EINVAL);
6070         }
6071
6072         return 0;
6073 }
6074
6075 static uint32_t
6076 instr_compact(struct instruction *instructions,
6077               struct instruction_data *instruction_data,
6078               uint32_t n_instructions)
6079 {
6080         uint32_t i, pos = 0;
6081
6082         /* Eliminate the invalid instructions that have been optimized out. */
6083         for (i = 0; i < n_instructions; i++) {
6084                 struct instruction *instr = &instructions[i];
6085                 struct instruction_data *data = &instruction_data[i];
6086
6087                 if (data->invalid)
6088                         continue;
6089
6090                 if (i != pos) {
6091                         memcpy(&instructions[pos], instr, sizeof(*instr));
6092                         memcpy(&instruction_data[pos], data, sizeof(*data));
6093                 }
6094
6095                 pos++;
6096         }
6097
6098         return pos;
6099 }
6100
6101 static int
6102 instr_pattern_extract_many_search(struct instruction *instr,
6103                                   struct instruction_data *data,
6104                                   uint32_t n_instr,
6105                                   uint32_t *n_pattern_instr)
6106 {
6107         uint32_t i;
6108
6109         for (i = 0; i < n_instr; i++) {
6110                 if (data[i].invalid)
6111                         break;
6112
6113                 if (instr[i].type != INSTR_HDR_EXTRACT)
6114                         break;
6115
6116                 if (i == RTE_DIM(instr->io.hdr.header_id))
6117                         break;
6118
6119                 if (i && data[i].n_users)
6120                         break;
6121         }
6122
6123         if (i < 2)
6124                 return 0;
6125
6126         *n_pattern_instr = i;
6127         return 1;
6128 }
6129
6130 static void
6131 instr_pattern_extract_many_replace(struct instruction *instr,
6132                                    struct instruction_data *data,
6133                                    uint32_t n_instr)
6134 {
6135         uint32_t i;
6136
6137         for (i = 1; i < n_instr; i++) {
6138                 instr[0].type++;
6139                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6140                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6141                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6142
6143                 data[i].invalid = 1;
6144         }
6145 }
6146
6147 static uint32_t
6148 instr_pattern_extract_many_optimize(struct instruction *instructions,
6149                                     struct instruction_data *instruction_data,
6150                                     uint32_t n_instructions)
6151 {
6152         uint32_t i;
6153
6154         for (i = 0; i < n_instructions; ) {
6155                 struct instruction *instr = &instructions[i];
6156                 struct instruction_data *data = &instruction_data[i];
6157                 uint32_t n_instr = 0;
6158                 int detected;
6159
6160                 /* Extract many. */
6161                 detected = instr_pattern_extract_many_search(instr,
6162                                                              data,
6163                                                              n_instructions - i,
6164                                                              &n_instr);
6165                 if (detected) {
6166                         instr_pattern_extract_many_replace(instr,
6167                                                            data,
6168                                                            n_instr);
6169                         i += n_instr;
6170                         continue;
6171                 }
6172
6173                 /* No pattern starting at the current instruction. */
6174                 i++;
6175         }
6176
6177         /* Eliminate the invalid instructions that have been optimized out. */
6178         n_instructions = instr_compact(instructions,
6179                                        instruction_data,
6180                                        n_instructions);
6181
6182         return n_instructions;
6183 }
6184
6185 static int
6186 instr_pattern_emit_many_tx_search(struct instruction *instr,
6187                                   struct instruction_data *data,
6188                                   uint32_t n_instr,
6189                                   uint32_t *n_pattern_instr)
6190 {
6191         uint32_t i;
6192
6193         for (i = 0; i < n_instr; i++) {
6194                 if (data[i].invalid)
6195                         break;
6196
6197                 if (instr[i].type != INSTR_HDR_EMIT)
6198                         break;
6199
6200                 if (i == RTE_DIM(instr->io.hdr.header_id))
6201                         break;
6202
6203                 if (i && data[i].n_users)
6204                         break;
6205         }
6206
6207         if (!i)
6208                 return 0;
6209
6210         if (!instruction_is_tx(instr[i].type))
6211                 return 0;
6212
6213         if (data[i].n_users)
6214                 return 0;
6215
6216         i++;
6217
6218         *n_pattern_instr = i;
6219         return 1;
6220 }
6221
6222 static void
6223 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6224                                    struct instruction_data *data,
6225                                    uint32_t n_instr)
6226 {
6227         uint32_t i;
6228
6229         /* Any emit instruction in addition to the first one. */
6230         for (i = 1; i < n_instr - 1; i++) {
6231                 instr[0].type++;
6232                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6233                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6234                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6235
6236                 data[i].invalid = 1;
6237         }
6238
6239         /* The TX instruction is the last one in the pattern. */
6240         instr[0].type++;
6241         instr[0].io.io.offset = instr[i].io.io.offset;
6242         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6243         data[i].invalid = 1;
6244 }
6245
6246 static uint32_t
6247 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6248                                     struct instruction_data *instruction_data,
6249                                     uint32_t n_instructions)
6250 {
6251         uint32_t i;
6252
6253         for (i = 0; i < n_instructions; ) {
6254                 struct instruction *instr = &instructions[i];
6255                 struct instruction_data *data = &instruction_data[i];
6256                 uint32_t n_instr = 0;
6257                 int detected;
6258
6259                 /* Emit many + TX. */
6260                 detected = instr_pattern_emit_many_tx_search(instr,
6261                                                              data,
6262                                                              n_instructions - i,
6263                                                              &n_instr);
6264                 if (detected) {
6265                         instr_pattern_emit_many_tx_replace(instr,
6266                                                            data,
6267                                                            n_instr);
6268                         i += n_instr;
6269                         continue;
6270                 }
6271
6272                 /* No pattern starting at the current instruction. */
6273                 i++;
6274         }
6275
6276         /* Eliminate the invalid instructions that have been optimized out. */
6277         n_instructions = instr_compact(instructions,
6278                                        instruction_data,
6279                                        n_instructions);
6280
6281         return n_instructions;
6282 }
6283
6284 static uint32_t
6285 action_arg_src_mov_count(struct action *a,
6286                          uint32_t arg_id,
6287                          struct instruction *instructions,
6288                          struct instruction_data *instruction_data,
6289                          uint32_t n_instructions);
6290
6291 static int
6292 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6293                                       struct action *a,
6294                                       struct instruction *instr,
6295                                       struct instruction_data *data,
6296                                       uint32_t n_instr,
6297                                       struct instruction *instructions,
6298                                       struct instruction_data *instruction_data,
6299                                       uint32_t n_instructions,
6300                                       uint32_t *n_pattern_instr)
6301 {
6302         struct header *h;
6303         uint32_t src_field_id, i, j;
6304
6305         /* Prerequisites. */
6306         if (!a || !a->st)
6307                 return 0;
6308
6309         /* First instruction: MOV_HM. */
6310         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6311                 return 0;
6312
6313         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6314         if (!h || h->st->var_size)
6315                 return 0;
6316
6317         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6318                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6319                         break;
6320
6321         if (src_field_id == a->st->n_fields)
6322                 return 0;
6323
6324         if (instr[0].mov.dst.offset ||
6325             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6326             instr[0].mov.src.struct_id ||
6327             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6328             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6329                 return 0;
6330
6331         if ((n_instr < h->st->n_fields + 1) ||
6332              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6333                 return 0;
6334
6335         /* Subsequent instructions: MOV_HM. */
6336         for (i = 1; i < h->st->n_fields; i++)
6337                 if (data[i].invalid ||
6338                     data[i].n_users ||
6339                     (instr[i].type != INSTR_MOV_HM) ||
6340                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6341                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6342                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6343                     instr[i].mov.src.struct_id ||
6344                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6345                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6346                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6347                         return 0;
6348
6349         /* Last instruction: HDR_VALIDATE. */
6350         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6351             (instr[i].valid.header_id != h->id))
6352                 return 0;
6353
6354         /* Check that none of the action args that are used as source for this
6355          * DMA transfer are not used as source in any other mov instruction.
6356          */
6357         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6358                 uint32_t n_users;
6359
6360                 n_users = action_arg_src_mov_count(a,
6361                                                    j,
6362                                                    instructions,
6363                                                    instruction_data,
6364                                                    n_instructions);
6365                 if (n_users > 1)
6366                         return 0;
6367         }
6368
6369         *n_pattern_instr = 1 + i;
6370         return 1;
6371 }
6372
6373 static void
6374 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6375                                        struct action *a,
6376                                        struct instruction *instr,
6377                                        struct instruction_data *data,
6378                                        uint32_t n_instr)
6379 {
6380         struct header *h;
6381         uint32_t src_field_id, src_offset, i;
6382
6383         /* Read from the instructions before they are modified. */
6384         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6385         if (!h)
6386                 return;
6387
6388         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6389                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6390                         break;
6391
6392         if (src_field_id == a->st->n_fields)
6393                 return;
6394
6395         src_offset = instr[0].mov.src.offset;
6396
6397         /* Modify the instructions. */
6398         instr[0].type = INSTR_DMA_HT;
6399         instr[0].dma.dst.header_id[0] = h->id;
6400         instr[0].dma.dst.struct_id[0] = h->struct_id;
6401         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6402         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6403
6404         for (i = 1; i < n_instr; i++)
6405                 data[i].invalid = 1;
6406
6407         /* Update the endianness of the action arguments to header endianness. */
6408         for (i = 0; i < h->st->n_fields; i++)
6409                 a->args_endianness[src_field_id + i] = 1;
6410 }
6411
6412 static uint32_t
6413 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6414                                         struct action *a,
6415                                         struct instruction *instructions,
6416                                         struct instruction_data *instruction_data,
6417                                         uint32_t n_instructions)
6418 {
6419         uint32_t i;
6420
6421         if (!a || !a->st)
6422                 return n_instructions;
6423
6424         for (i = 0; i < n_instructions; ) {
6425                 struct instruction *instr = &instructions[i];
6426                 struct instruction_data *data = &instruction_data[i];
6427                 uint32_t n_instr = 0;
6428                 int detected;
6429
6430                 /* Mov all + validate. */
6431                 detected = instr_pattern_mov_all_validate_search(p,
6432                                                                  a,
6433                                                                  instr,
6434                                                                  data,
6435                                                                  n_instructions - i,
6436                                                                  instructions,
6437                                                                  instruction_data,
6438                                                                  n_instructions,
6439                                                                  &n_instr);
6440                 if (detected) {
6441                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6442                         i += n_instr;
6443                         continue;
6444                 }
6445
6446                 /* No pattern starting at the current instruction. */
6447                 i++;
6448         }
6449
6450         /* Eliminate the invalid instructions that have been optimized out. */
6451         n_instructions = instr_compact(instructions,
6452                                        instruction_data,
6453                                        n_instructions);
6454
6455         return n_instructions;
6456 }
6457
6458 static int
6459 instr_pattern_dma_many_search(struct instruction *instr,
6460                               struct instruction_data *data,
6461                               uint32_t n_instr,
6462                               uint32_t *n_pattern_instr)
6463 {
6464         uint32_t i;
6465
6466         for (i = 0; i < n_instr; i++) {
6467                 if (data[i].invalid)
6468                         break;
6469
6470                 if (instr[i].type != INSTR_DMA_HT)
6471                         break;
6472
6473                 if (i == RTE_DIM(instr->dma.dst.header_id))
6474                         break;
6475
6476                 if (i && data[i].n_users)
6477                         break;
6478         }
6479
6480         if (i < 2)
6481                 return 0;
6482
6483         *n_pattern_instr = i;
6484         return 1;
6485 }
6486
6487 static void
6488 instr_pattern_dma_many_replace(struct instruction *instr,
6489                                struct instruction_data *data,
6490                                uint32_t n_instr)
6491 {
6492         uint32_t i;
6493
6494         for (i = 1; i < n_instr; i++) {
6495                 instr[0].type++;
6496                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6497                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6498                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6499                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6500
6501                 data[i].invalid = 1;
6502         }
6503 }
6504
6505 static uint32_t
6506 instr_pattern_dma_many_optimize(struct instruction *instructions,
6507                struct instruction_data *instruction_data,
6508                uint32_t n_instructions)
6509 {
6510         uint32_t i;
6511
6512         for (i = 0; i < n_instructions; ) {
6513                 struct instruction *instr = &instructions[i];
6514                 struct instruction_data *data = &instruction_data[i];
6515                 uint32_t n_instr = 0;
6516                 int detected;
6517
6518                 /* DMA many. */
6519                 detected = instr_pattern_dma_many_search(instr,
6520                                                          data,
6521                                                          n_instructions - i,
6522                                                          &n_instr);
6523                 if (detected) {
6524                         instr_pattern_dma_many_replace(instr, data, n_instr);
6525                         i += n_instr;
6526                         continue;
6527                 }
6528
6529                 /* No pattern starting at the current instruction. */
6530                 i++;
6531         }
6532
6533         /* Eliminate the invalid instructions that have been optimized out. */
6534         n_instructions = instr_compact(instructions,
6535                                        instruction_data,
6536                                        n_instructions);
6537
6538         return n_instructions;
6539 }
6540
6541 static uint32_t
6542 instr_optimize(struct rte_swx_pipeline *p,
6543                struct action *a,
6544                struct instruction *instructions,
6545                struct instruction_data *instruction_data,
6546                uint32_t n_instructions)
6547 {
6548         /* Extract many. */
6549         n_instructions = instr_pattern_extract_many_optimize(instructions,
6550                                                              instruction_data,
6551                                                              n_instructions);
6552
6553         /* Emit many + TX. */
6554         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6555                                                              instruction_data,
6556                                                              n_instructions);
6557
6558         /* Mov all + validate. */
6559         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6560                                                                  a,
6561                                                                  instructions,
6562                                                                  instruction_data,
6563                                                                  n_instructions);
6564
6565         /* DMA many. */
6566         n_instructions = instr_pattern_dma_many_optimize(instructions,
6567                                                          instruction_data,
6568                                                          n_instructions);
6569
6570         return n_instructions;
6571 }
6572
6573 static int
6574 instruction_config(struct rte_swx_pipeline *p,
6575                    struct action *a,
6576                    const char **instructions,
6577                    uint32_t n_instructions)
6578 {
6579         struct instruction *instr = NULL;
6580         struct instruction_data *data = NULL;
6581         int err = 0;
6582         uint32_t i;
6583
6584         CHECK(n_instructions, EINVAL);
6585         CHECK(instructions, EINVAL);
6586         for (i = 0; i < n_instructions; i++)
6587                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6588
6589         /* Memory allocation. */
6590         instr = calloc(n_instructions, sizeof(struct instruction));
6591         if (!instr) {
6592                 err = -ENOMEM;
6593                 goto error;
6594         }
6595
6596         data = calloc(n_instructions, sizeof(struct instruction_data));
6597         if (!data) {
6598                 err = -ENOMEM;
6599                 goto error;
6600         }
6601
6602         for (i = 0; i < n_instructions; i++) {
6603                 char *string = strdup(instructions[i]);
6604                 if (!string) {
6605                         err = -ENOMEM;
6606                         goto error;
6607                 }
6608
6609                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6610                 if (err) {
6611                         free(string);
6612                         goto error;
6613                 }
6614
6615                 free(string);
6616         }
6617
6618         err = instr_label_check(data, n_instructions);
6619         if (err)
6620                 goto error;
6621
6622         err = instr_verify(p, a, instr, data, n_instructions);
6623         if (err)
6624                 goto error;
6625
6626         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6627
6628         err = instr_jmp_resolve(instr, data, n_instructions);
6629         if (err)
6630                 goto error;
6631
6632         if (a) {
6633                 a->instructions = instr;
6634                 a->instruction_data = data;
6635                 a->n_instructions = n_instructions;
6636         } else {
6637                 p->instructions = instr;
6638                 p->instruction_data = data;
6639                 p->n_instructions = n_instructions;
6640         }
6641
6642         return 0;
6643
6644 error:
6645         free(data);
6646         free(instr);
6647         return err;
6648 }
6649
6650 static instr_exec_t instruction_table[] = {
6651         [INSTR_RX] = instr_rx_exec,
6652         [INSTR_TX] = instr_tx_exec,
6653         [INSTR_TX_I] = instr_tx_i_exec,
6654
6655         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6656         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6657         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6658         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6659         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6660         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6661         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6662         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6663         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6664         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6665
6666         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6667         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6668         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6669         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6670         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6671         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6672         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6673         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6674         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6675
6676         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6677         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6678
6679         [INSTR_MOV] = instr_mov_exec,
6680         [INSTR_MOV_MH] = instr_mov_mh_exec,
6681         [INSTR_MOV_HM] = instr_mov_hm_exec,
6682         [INSTR_MOV_HH] = instr_mov_hh_exec,
6683         [INSTR_MOV_I] = instr_mov_i_exec,
6684
6685         [INSTR_DMA_HT] = instr_dma_ht_exec,
6686         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6687         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6688         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6689         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6690         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6691         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6692         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6693
6694         [INSTR_ALU_ADD] = instr_alu_add_exec,
6695         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6696         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6697         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6698         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6699         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6700
6701         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6702         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6703         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6704         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6705         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6706         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6707
6708         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6709         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6710         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6711         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6712
6713         [INSTR_ALU_AND] = instr_alu_and_exec,
6714         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6715         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6716         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6717         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6718
6719         [INSTR_ALU_OR] = instr_alu_or_exec,
6720         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6721         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6722         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6723         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6724
6725         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6726         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6727         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6728         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6729         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6730
6731         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6732         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6733         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6734         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6735         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6736         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6737
6738         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6739         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6740         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6741         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6742         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6743         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6744
6745         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6746         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6747         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6748
6749         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6750         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6751         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6752         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6753         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6754         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6755
6756         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6757         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6758         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6759         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6760         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6761         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6762         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6763         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6764         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6765
6766         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6767         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6768         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6769         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6770         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6771         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6772         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6773         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6774         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6775
6776         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6777         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6778         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6779
6780         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6781         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6782         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6783         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6784         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6785         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6786         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6787         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6788         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6789         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6790         [INSTR_METER_IMM] = instr_meter_imm_exec,
6791         [INSTR_METER_IMI] = instr_meter_imi_exec,
6792
6793         [INSTR_TABLE] = instr_table_exec,
6794         [INSTR_TABLE_AF] = instr_table_af_exec,
6795         [INSTR_SELECTOR] = instr_selector_exec,
6796         [INSTR_LEARNER] = instr_learner_exec,
6797         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6798         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6799         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6800         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6801         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6802
6803         [INSTR_JMP] = instr_jmp_exec,
6804         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6805         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6806         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6807         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6808         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6809         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6810
6811         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6812         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6813         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6814         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6815         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6816
6817         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6818         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6819         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6820         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6821         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6822
6823         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6824         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6825         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6826         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6827         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6828         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6829
6830         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6831         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6832         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6833         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6834         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6835         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6836
6837         [INSTR_RETURN] = instr_return_exec,
6838 };
6839
6840 static int
6841 instruction_table_build(struct rte_swx_pipeline *p)
6842 {
6843         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6844                                       sizeof(struct instr_exec_t *));
6845         if (!p->instruction_table)
6846                 return -EINVAL;
6847
6848         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6849
6850         return 0;
6851 }
6852
6853 static void
6854 instruction_table_build_free(struct rte_swx_pipeline *p)
6855 {
6856         if (!p->instruction_table)
6857                 return;
6858
6859         free(p->instruction_table);
6860         p->instruction_table = NULL;
6861 }
6862
6863 static void
6864 instruction_table_free(struct rte_swx_pipeline *p)
6865 {
6866         instruction_table_build_free(p);
6867 }
6868
6869 static inline void
6870 instr_exec(struct rte_swx_pipeline *p)
6871 {
6872         struct thread *t = &p->threads[p->thread_id];
6873         struct instruction *ip = t->ip;
6874         instr_exec_t instr = p->instruction_table[ip->type];
6875
6876         instr(p);
6877 }
6878
6879 /*
6880  * Action.
6881  */
6882 static struct action *
6883 action_find(struct rte_swx_pipeline *p, const char *name)
6884 {
6885         struct action *elem;
6886
6887         if (!name)
6888                 return NULL;
6889
6890         TAILQ_FOREACH(elem, &p->actions, node)
6891                 if (strcmp(elem->name, name) == 0)
6892                         return elem;
6893
6894         return NULL;
6895 }
6896
6897 static struct action *
6898 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6899 {
6900         struct action *action = NULL;
6901
6902         TAILQ_FOREACH(action, &p->actions, node)
6903                 if (action->id == id)
6904                         return action;
6905
6906         return NULL;
6907 }
6908
6909 static struct field *
6910 action_field_find(struct action *a, const char *name)
6911 {
6912         return a->st ? struct_type_field_find(a->st, name) : NULL;
6913 }
6914
6915 static struct field *
6916 action_field_parse(struct action *action, const char *name)
6917 {
6918         if (name[0] != 't' || name[1] != '.')
6919                 return NULL;
6920
6921         return action_field_find(action, &name[2]);
6922 }
6923
6924 static int
6925 action_has_nbo_args(struct action *a)
6926 {
6927         uint32_t i;
6928
6929         /* Return if the action does not have any args. */
6930         if (!a->st)
6931                 return 0; /* FALSE */
6932
6933         for (i = 0; i < a->st->n_fields; i++)
6934                 if (a->args_endianness[i])
6935                         return 1; /* TRUE */
6936
6937         return 0; /* FALSE */
6938 }
6939
6940 static int
6941 action_does_learning(struct action *a)
6942 {
6943         uint32_t i;
6944
6945         for (i = 0; i < a->n_instructions; i++)
6946                 switch (a->instructions[i].type) {
6947                 case INSTR_LEARNER_LEARN:
6948                         return 1; /* TRUE */
6949
6950                 case INSTR_LEARNER_FORGET:
6951                         return 1; /* TRUE */
6952
6953                 default:
6954                         continue;
6955                 }
6956
6957         return 0; /* FALSE */
6958 }
6959
6960 int
6961 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6962                                const char *name,
6963                                const char *args_struct_type_name,
6964                                const char **instructions,
6965                                uint32_t n_instructions)
6966 {
6967         struct struct_type *args_struct_type = NULL;
6968         struct action *a;
6969         int err;
6970
6971         CHECK(p, EINVAL);
6972
6973         CHECK_NAME(name, EINVAL);
6974         CHECK(!action_find(p, name), EEXIST);
6975
6976         if (args_struct_type_name) {
6977                 CHECK_NAME(args_struct_type_name, EINVAL);
6978                 args_struct_type = struct_type_find(p, args_struct_type_name);
6979                 CHECK(args_struct_type, EINVAL);
6980                 CHECK(!args_struct_type->var_size, EINVAL);
6981         }
6982
6983         /* Node allocation. */
6984         a = calloc(1, sizeof(struct action));
6985         CHECK(a, ENOMEM);
6986         if (args_struct_type) {
6987                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6988                 if (!a->args_endianness) {
6989                         free(a);
6990                         CHECK(0, ENOMEM);
6991                 }
6992         }
6993
6994         /* Node initialization. */
6995         strcpy(a->name, name);
6996         a->st = args_struct_type;
6997         a->id = p->n_actions;
6998
6999         /* Instruction translation. */
7000         err = instruction_config(p, a, instructions, n_instructions);
7001         if (err) {
7002                 free(a->args_endianness);
7003                 free(a);
7004                 return err;
7005         }
7006
7007         /* Node add to tailq. */
7008         TAILQ_INSERT_TAIL(&p->actions, a, node);
7009         p->n_actions++;
7010
7011         return 0;
7012 }
7013
7014 static int
7015 action_build(struct rte_swx_pipeline *p)
7016 {
7017         struct action *action;
7018
7019         /* p->action_instructions. */
7020         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7021         CHECK(p->action_instructions, ENOMEM);
7022
7023         TAILQ_FOREACH(action, &p->actions, node)
7024                 p->action_instructions[action->id] = action->instructions;
7025
7026         /* p->action_funcs. */
7027         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7028         CHECK(p->action_funcs, ENOMEM);
7029
7030         return 0;
7031 }
7032
7033 static void
7034 action_build_free(struct rte_swx_pipeline *p)
7035 {
7036         free(p->action_funcs);
7037         p->action_funcs = NULL;
7038
7039         free(p->action_instructions);
7040         p->action_instructions = NULL;
7041 }
7042
7043 static void
7044 action_free(struct rte_swx_pipeline *p)
7045 {
7046         action_build_free(p);
7047
7048         for ( ; ; ) {
7049                 struct action *action;
7050
7051                 action = TAILQ_FIRST(&p->actions);
7052                 if (!action)
7053                         break;
7054
7055                 TAILQ_REMOVE(&p->actions, action, node);
7056                 free(action->instruction_data);
7057                 free(action->instructions);
7058                 free(action);
7059         }
7060 }
7061
7062 static uint32_t
7063 action_arg_src_mov_count(struct action *a,
7064                          uint32_t arg_id,
7065                          struct instruction *instructions,
7066                          struct instruction_data *instruction_data,
7067                          uint32_t n_instructions)
7068 {
7069         uint32_t offset, n_users = 0, i;
7070
7071         if (!a->st ||
7072             (arg_id >= a->st->n_fields) ||
7073             !instructions ||
7074             !instruction_data ||
7075             !n_instructions)
7076                 return 0;
7077
7078         offset = a->st->fields[arg_id].offset / 8;
7079
7080         for (i = 0; i < n_instructions; i++) {
7081                 struct instruction *instr = &instructions[i];
7082                 struct instruction_data *data = &instruction_data[i];
7083
7084                 if (data->invalid ||
7085                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7086                     instr->mov.src.struct_id ||
7087                     (instr->mov.src.offset != offset))
7088                         continue;
7089
7090                 n_users++;
7091         }
7092
7093         return n_users;
7094 }
7095
7096 /*
7097  * Table.
7098  */
7099 static struct table_type *
7100 table_type_find(struct rte_swx_pipeline *p, const char *name)
7101 {
7102         struct table_type *elem;
7103
7104         TAILQ_FOREACH(elem, &p->table_types, node)
7105                 if (strcmp(elem->name, name) == 0)
7106                         return elem;
7107
7108         return NULL;
7109 }
7110
7111 static struct table_type *
7112 table_type_resolve(struct rte_swx_pipeline *p,
7113                    const char *recommended_type_name,
7114                    enum rte_swx_table_match_type match_type)
7115 {
7116         struct table_type *elem;
7117
7118         /* Only consider the recommended type if the match type is correct. */
7119         if (recommended_type_name)
7120                 TAILQ_FOREACH(elem, &p->table_types, node)
7121                         if (!strcmp(elem->name, recommended_type_name) &&
7122                             (elem->match_type == match_type))
7123                                 return elem;
7124
7125         /* Ignore the recommended type and get the first element with this match
7126          * type.
7127          */
7128         TAILQ_FOREACH(elem, &p->table_types, node)
7129                 if (elem->match_type == match_type)
7130                         return elem;
7131
7132         return NULL;
7133 }
7134
7135 static struct table *
7136 table_find(struct rte_swx_pipeline *p, const char *name)
7137 {
7138         struct table *elem;
7139
7140         TAILQ_FOREACH(elem, &p->tables, node)
7141                 if (strcmp(elem->name, name) == 0)
7142                         return elem;
7143
7144         return NULL;
7145 }
7146
7147 static struct table *
7148 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7149 {
7150         struct table *table = NULL;
7151
7152         TAILQ_FOREACH(table, &p->tables, node)
7153                 if (table->id == id)
7154                         return table;
7155
7156         return NULL;
7157 }
7158
7159 int
7160 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7161                                      const char *name,
7162                                      enum rte_swx_table_match_type match_type,
7163                                      struct rte_swx_table_ops *ops)
7164 {
7165         struct table_type *elem;
7166
7167         CHECK(p, EINVAL);
7168
7169         CHECK_NAME(name, EINVAL);
7170         CHECK(!table_type_find(p, name), EEXIST);
7171
7172         CHECK(ops, EINVAL);
7173         CHECK(ops->create, EINVAL);
7174         CHECK(ops->lkp, EINVAL);
7175         CHECK(ops->free, EINVAL);
7176
7177         /* Node allocation. */
7178         elem = calloc(1, sizeof(struct table_type));
7179         CHECK(elem, ENOMEM);
7180
7181         /* Node initialization. */
7182         strcpy(elem->name, name);
7183         elem->match_type = match_type;
7184         memcpy(&elem->ops, ops, sizeof(*ops));
7185
7186         /* Node add to tailq. */
7187         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7188
7189         return 0;
7190 }
7191
7192 static int
7193 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7194                          uint32_t n_fields,
7195                          enum rte_swx_table_match_type *match_type)
7196 {
7197         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7198
7199         for (i = 0; i < n_fields; i++) {
7200                 struct rte_swx_match_field_params  *f = &fields[i];
7201
7202                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7203                         n_fields_em++;
7204
7205                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7206                         n_fields_lpm++;
7207         }
7208
7209         if ((n_fields_lpm > 1) ||
7210             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7211                 return -EINVAL;
7212
7213         *match_type = (n_fields_em == n_fields) ?
7214                        RTE_SWX_TABLE_MATCH_EXACT :
7215                        RTE_SWX_TABLE_MATCH_WILDCARD;
7216
7217         return 0;
7218 }
7219
7220 static int
7221 table_match_fields_check(struct rte_swx_pipeline *p,
7222                          struct rte_swx_pipeline_table_params *params,
7223                          struct header **header)
7224 {
7225         struct header *h0 = NULL;
7226         struct field *hf, *mf;
7227         uint32_t *offset = NULL, i;
7228         int status = 0;
7229
7230         /* Return if no match fields. */
7231         if (!params->n_fields) {
7232                 if (params->fields) {
7233                         status = -EINVAL;
7234                         goto end;
7235                 }
7236
7237                 if (header)
7238                         *header = NULL;
7239
7240                 return 0;
7241         }
7242
7243         /* Memory allocation. */
7244         offset = calloc(params->n_fields, sizeof(uint32_t));
7245         if (!offset) {
7246                 status = -ENOMEM;
7247                 goto end;
7248         }
7249
7250         /* Check that all the match fields belong to either the same header or
7251          * to the meta-data.
7252          */
7253         hf = header_field_parse(p, params->fields[0].name, &h0);
7254         mf = metadata_field_parse(p, params->fields[0].name);
7255         if ((!hf && !mf) || (hf && hf->var_size)) {
7256                 status = -EINVAL;
7257                 goto end;
7258         }
7259
7260         offset[0] = h0 ? hf->offset : mf->offset;
7261
7262         for (i = 1; i < params->n_fields; i++)
7263                 if (h0) {
7264                         struct header *h;
7265
7266                         hf = header_field_parse(p, params->fields[i].name, &h);
7267                         if (!hf || (h->id != h0->id) || hf->var_size) {
7268                                 status = -EINVAL;
7269                                 goto end;
7270                         }
7271
7272                         offset[i] = hf->offset;
7273                 } else {
7274                         mf = metadata_field_parse(p, params->fields[i].name);
7275                         if (!mf) {
7276                                 status = -EINVAL;
7277                                 goto end;
7278                         }
7279
7280                         offset[i] = mf->offset;
7281                 }
7282
7283         /* Check that there are no duplicated match fields. */
7284         for (i = 0; i < params->n_fields; i++) {
7285                 uint32_t j;
7286
7287                 for (j = 0; j < i; j++)
7288                         if (offset[j] == offset[i]) {
7289                                 status = -EINVAL;
7290                                 goto end;
7291                         }
7292         }
7293
7294         /* Return. */
7295         if (header)
7296                 *header = h0;
7297
7298 end:
7299         free(offset);
7300         return status;
7301 }
7302
7303 int
7304 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7305                               const char *name,
7306                               struct rte_swx_pipeline_table_params *params,
7307                               const char *recommended_table_type_name,
7308                               const char *args,
7309                               uint32_t size)
7310 {
7311         struct table_type *type;
7312         struct table *t = NULL;
7313         struct action *default_action;
7314         struct header *header = NULL;
7315         uint32_t action_data_size_max = 0, i;
7316         int status = 0;
7317
7318         CHECK(p, EINVAL);
7319
7320         CHECK_NAME(name, EINVAL);
7321         CHECK(!table_find(p, name), EEXIST);
7322         CHECK(!selector_find(p, name), EEXIST);
7323         CHECK(!learner_find(p, name), EEXIST);
7324
7325         CHECK(params, EINVAL);
7326
7327         /* Match checks. */
7328         status = table_match_fields_check(p, params, &header);
7329         if (status)
7330                 return status;
7331
7332         /* Action checks. */
7333         CHECK(params->n_actions, EINVAL);
7334         CHECK(params->action_names, EINVAL);
7335         for (i = 0; i < params->n_actions; i++) {
7336                 const char *action_name = params->action_names[i];
7337                 struct action *a;
7338                 uint32_t action_data_size;
7339                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7340
7341                 CHECK_NAME(action_name, EINVAL);
7342
7343                 a = action_find(p, action_name);
7344                 CHECK(a, EINVAL);
7345                 CHECK(!action_does_learning(a), EINVAL);
7346
7347                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7348                 if (action_data_size > action_data_size_max)
7349                         action_data_size_max = action_data_size;
7350
7351                 if (params->action_is_for_table_entries)
7352                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7353                 if (params->action_is_for_default_entry)
7354                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7355                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7356         }
7357
7358         CHECK_NAME(params->default_action_name, EINVAL);
7359         for (i = 0; i < p->n_actions; i++)
7360                 if (!strcmp(params->action_names[i],
7361                             params->default_action_name))
7362                         break;
7363         CHECK(i < params->n_actions, EINVAL);
7364         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7365               EINVAL);
7366
7367         default_action = action_find(p, params->default_action_name);
7368         CHECK((default_action->st && params->default_action_data) ||
7369               !params->default_action_data, EINVAL);
7370
7371         /* Table type checks. */
7372         if (recommended_table_type_name)
7373                 CHECK_NAME(recommended_table_type_name, EINVAL);
7374
7375         if (params->n_fields) {
7376                 enum rte_swx_table_match_type match_type;
7377
7378                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7379                 if (status)
7380                         return status;
7381
7382                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7383                 CHECK(type, EINVAL);
7384         } else {
7385                 type = NULL;
7386         }
7387
7388         /* Memory allocation. */
7389         t = calloc(1, sizeof(struct table));
7390         CHECK(t, ENOMEM);
7391
7392         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7393         if (!t->fields)
7394                 goto nomem;
7395
7396         t->actions = calloc(params->n_actions, sizeof(struct action *));
7397         if (!t->actions)
7398                 goto nomem;
7399
7400         if (action_data_size_max) {
7401                 t->default_action_data = calloc(1, action_data_size_max);
7402                 if (!t->default_action_data)
7403                         goto nomem;
7404         }
7405
7406         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7407         if (!t->action_is_for_table_entries)
7408                 goto nomem;
7409
7410         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7411         if (!t->action_is_for_default_entry)
7412                 goto nomem;
7413
7414         /* Node initialization. */
7415         strcpy(t->name, name);
7416         if (args && args[0])
7417                 strcpy(t->args, args);
7418         t->type = type;
7419
7420         for (i = 0; i < params->n_fields; i++) {
7421                 struct rte_swx_match_field_params *field = &params->fields[i];
7422                 struct match_field *f = &t->fields[i];
7423
7424                 f->match_type = field->match_type;
7425                 f->field = header ?
7426                         header_field_parse(p, field->name, NULL) :
7427                         metadata_field_parse(p, field->name);
7428         }
7429         t->n_fields = params->n_fields;
7430         t->header = header;
7431
7432         for (i = 0; i < params->n_actions; i++) {
7433                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7434
7435                 if (params->action_is_for_table_entries)
7436                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7437                 if (params->action_is_for_default_entry)
7438                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7439
7440                 t->actions[i] = action_find(p, params->action_names[i]);
7441                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7442                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7443         }
7444         t->default_action = default_action;
7445         if (default_action->st)
7446                 memcpy(t->default_action_data,
7447                        params->default_action_data,
7448                        default_action->st->n_bits / 8);
7449         t->n_actions = params->n_actions;
7450         t->default_action_is_const = params->default_action_is_const;
7451         t->action_data_size_max = action_data_size_max;
7452
7453         t->size = size;
7454         t->id = p->n_tables;
7455
7456         /* Node add to tailq. */
7457         TAILQ_INSERT_TAIL(&p->tables, t, node);
7458         p->n_tables++;
7459
7460         return 0;
7461
7462 nomem:
7463         if (!t)
7464                 return -ENOMEM;
7465
7466         free(t->action_is_for_default_entry);
7467         free(t->action_is_for_table_entries);
7468         free(t->default_action_data);
7469         free(t->actions);
7470         free(t->fields);
7471         free(t);
7472
7473         return -ENOMEM;
7474 }
7475
7476 static struct rte_swx_table_params *
7477 table_params_get(struct table *table)
7478 {
7479         struct rte_swx_table_params *params;
7480         struct field *first, *last;
7481         uint8_t *key_mask;
7482         uint32_t key_size, key_offset, action_data_size, i;
7483
7484         /* Memory allocation. */
7485         params = calloc(1, sizeof(struct rte_swx_table_params));
7486         if (!params)
7487                 return NULL;
7488
7489         /* Find first (smallest offset) and last (biggest offset) match fields. */
7490         first = table->fields[0].field;
7491         last = table->fields[0].field;
7492
7493         for (i = 0; i < table->n_fields; i++) {
7494                 struct field *f = table->fields[i].field;
7495
7496                 if (f->offset < first->offset)
7497                         first = f;
7498
7499                 if (f->offset > last->offset)
7500                         last = f;
7501         }
7502
7503         /* Key offset and size. */
7504         key_offset = first->offset / 8;
7505         key_size = (last->offset + last->n_bits - first->offset) / 8;
7506
7507         /* Memory allocation. */
7508         key_mask = calloc(1, key_size);
7509         if (!key_mask) {
7510                 free(params);
7511                 return NULL;
7512         }
7513
7514         /* Key mask. */
7515         for (i = 0; i < table->n_fields; i++) {
7516                 struct field *f = table->fields[i].field;
7517                 uint32_t start = (f->offset - first->offset) / 8;
7518                 size_t size = f->n_bits / 8;
7519
7520                 memset(&key_mask[start], 0xFF, size);
7521         }
7522
7523         /* Action data size. */
7524         action_data_size = 0;
7525         for (i = 0; i < table->n_actions; i++) {
7526                 struct action *action = table->actions[i];
7527                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7528
7529                 if (ads > action_data_size)
7530                         action_data_size = ads;
7531         }
7532
7533         /* Fill in. */
7534         params->match_type = table->type->match_type;
7535         params->key_size = key_size;
7536         params->key_offset = key_offset;
7537         params->key_mask0 = key_mask;
7538         params->action_data_size = action_data_size;
7539         params->n_keys_max = table->size;
7540
7541         return params;
7542 }
7543
7544 static void
7545 table_params_free(struct rte_swx_table_params *params)
7546 {
7547         if (!params)
7548                 return;
7549
7550         free(params->key_mask0);
7551         free(params);
7552 }
7553
7554 static int
7555 table_stub_lkp(void *table __rte_unused,
7556                void *mailbox __rte_unused,
7557                uint8_t **key __rte_unused,
7558                uint64_t *action_id __rte_unused,
7559                uint8_t **action_data __rte_unused,
7560                int *hit)
7561 {
7562         *hit = 0;
7563         return 1; /* DONE. */
7564 }
7565
7566 static int
7567 table_build(struct rte_swx_pipeline *p)
7568 {
7569         uint32_t i;
7570
7571         /* Per pipeline: table statistics. */
7572         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7573         CHECK(p->table_stats, ENOMEM);
7574
7575         for (i = 0; i < p->n_tables; i++) {
7576                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7577                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7578         }
7579
7580         /* Per thread: table runt-time. */
7581         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7582                 struct thread *t = &p->threads[i];
7583                 struct table *table;
7584
7585                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7586                 CHECK(t->tables, ENOMEM);
7587
7588                 TAILQ_FOREACH(table, &p->tables, node) {
7589                         struct table_runtime *r = &t->tables[table->id];
7590
7591                         if (table->type) {
7592                                 uint64_t size;
7593
7594                                 size = table->type->ops.mailbox_size_get();
7595
7596                                 /* r->func. */
7597                                 r->func = table->type->ops.lkp;
7598
7599                                 /* r->mailbox. */
7600                                 if (size) {
7601                                         r->mailbox = calloc(1, size);
7602                                         CHECK(r->mailbox, ENOMEM);
7603                                 }
7604
7605                                 /* r->key. */
7606                                 r->key = table->header ?
7607                                         &t->structs[table->header->struct_id] :
7608                                         &t->structs[p->metadata_struct_id];
7609                         } else {
7610                                 r->func = table_stub_lkp;
7611                         }
7612                 }
7613         }
7614
7615         return 0;
7616 }
7617
7618 static void
7619 table_build_free(struct rte_swx_pipeline *p)
7620 {
7621         uint32_t i;
7622
7623         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7624                 struct thread *t = &p->threads[i];
7625                 uint32_t j;
7626
7627                 if (!t->tables)
7628                         continue;
7629
7630                 for (j = 0; j < p->n_tables; j++) {
7631                         struct table_runtime *r = &t->tables[j];
7632
7633                         free(r->mailbox);
7634                 }
7635
7636                 free(t->tables);
7637                 t->tables = NULL;
7638         }
7639
7640         if (p->table_stats) {
7641                 for (i = 0; i < p->n_tables; i++)
7642                         free(p->table_stats[i].n_pkts_action);
7643
7644                 free(p->table_stats);
7645         }
7646 }
7647
7648 static void
7649 table_free(struct rte_swx_pipeline *p)
7650 {
7651         table_build_free(p);
7652
7653         /* Tables. */
7654         for ( ; ; ) {
7655                 struct table *elem;
7656
7657                 elem = TAILQ_FIRST(&p->tables);
7658                 if (!elem)
7659                         break;
7660
7661                 TAILQ_REMOVE(&p->tables, elem, node);
7662                 free(elem->fields);
7663                 free(elem->actions);
7664                 free(elem->default_action_data);
7665                 free(elem);
7666         }
7667
7668         /* Table types. */
7669         for ( ; ; ) {
7670                 struct table_type *elem;
7671
7672                 elem = TAILQ_FIRST(&p->table_types);
7673                 if (!elem)
7674                         break;
7675
7676                 TAILQ_REMOVE(&p->table_types, elem, node);
7677                 free(elem);
7678         }
7679 }
7680
7681 /*
7682  * Selector.
7683  */
7684 static struct selector *
7685 selector_find(struct rte_swx_pipeline *p, const char *name)
7686 {
7687         struct selector *s;
7688
7689         TAILQ_FOREACH(s, &p->selectors, node)
7690                 if (strcmp(s->name, name) == 0)
7691                         return s;
7692
7693         return NULL;
7694 }
7695
7696 static struct selector *
7697 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7698 {
7699         struct selector *s = NULL;
7700
7701         TAILQ_FOREACH(s, &p->selectors, node)
7702                 if (s->id == id)
7703                         return s;
7704
7705         return NULL;
7706 }
7707
7708 static int
7709 selector_fields_check(struct rte_swx_pipeline *p,
7710                       struct rte_swx_pipeline_selector_params *params,
7711                       struct header **header)
7712 {
7713         struct header *h0 = NULL;
7714         struct field *hf, *mf;
7715         uint32_t i;
7716
7717         /* Return if no selector fields. */
7718         if (!params->n_selector_fields || !params->selector_field_names)
7719                 return -EINVAL;
7720
7721         /* Check that all the selector fields either belong to the same header
7722          * or are all meta-data fields.
7723          */
7724         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7725         mf = metadata_field_parse(p, params->selector_field_names[0]);
7726         if (!hf && !mf)
7727                 return -EINVAL;
7728
7729         for (i = 1; i < params->n_selector_fields; i++)
7730                 if (h0) {
7731                         struct header *h;
7732
7733                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7734                         if (!hf || (h->id != h0->id))
7735                                 return -EINVAL;
7736                 } else {
7737                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7738                         if (!mf)
7739                                 return -EINVAL;
7740                 }
7741
7742         /* Check that there are no duplicated match fields. */
7743         for (i = 0; i < params->n_selector_fields; i++) {
7744                 const char *field_name = params->selector_field_names[i];
7745                 uint32_t j;
7746
7747                 for (j = i + 1; j < params->n_selector_fields; j++)
7748                         if (!strcmp(params->selector_field_names[j], field_name))
7749                                 return -EINVAL;
7750         }
7751
7752         /* Return. */
7753         if (header)
7754                 *header = h0;
7755
7756         return 0;
7757 }
7758
7759 int
7760 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7761                                  const char *name,
7762                                  struct rte_swx_pipeline_selector_params *params)
7763 {
7764         struct selector *s;
7765         struct header *selector_header = NULL;
7766         struct field *group_id_field, *member_id_field;
7767         uint32_t i;
7768         int status = 0;
7769
7770         CHECK(p, EINVAL);
7771
7772         CHECK_NAME(name, EINVAL);
7773         CHECK(!table_find(p, name), EEXIST);
7774         CHECK(!selector_find(p, name), EEXIST);
7775         CHECK(!learner_find(p, name), EEXIST);
7776
7777         CHECK(params, EINVAL);
7778
7779         CHECK_NAME(params->group_id_field_name, EINVAL);
7780         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7781         CHECK(group_id_field, EINVAL);
7782
7783         for (i = 0; i < params->n_selector_fields; i++) {
7784                 const char *field_name = params->selector_field_names[i];
7785
7786                 CHECK_NAME(field_name, EINVAL);
7787         }
7788         status = selector_fields_check(p, params, &selector_header);
7789         if (status)
7790                 return status;
7791
7792         CHECK_NAME(params->member_id_field_name, EINVAL);
7793         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7794         CHECK(member_id_field, EINVAL);
7795
7796         CHECK(params->n_groups_max, EINVAL);
7797
7798         CHECK(params->n_members_per_group_max, EINVAL);
7799
7800         /* Memory allocation. */
7801         s = calloc(1, sizeof(struct selector));
7802         if (!s) {
7803                 status = -ENOMEM;
7804                 goto error;
7805         }
7806
7807         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7808         if (!s->selector_fields) {
7809                 status = -ENOMEM;
7810                 goto error;
7811         }
7812
7813         /* Node initialization. */
7814         strcpy(s->name, name);
7815
7816         s->group_id_field = group_id_field;
7817
7818         for (i = 0; i < params->n_selector_fields; i++) {
7819                 const char *field_name = params->selector_field_names[i];
7820
7821                 s->selector_fields[i] = selector_header ?
7822                         header_field_parse(p, field_name, NULL) :
7823                         metadata_field_parse(p, field_name);
7824         }
7825
7826         s->n_selector_fields = params->n_selector_fields;
7827
7828         s->selector_header = selector_header;
7829
7830         s->member_id_field = member_id_field;
7831
7832         s->n_groups_max = params->n_groups_max;
7833
7834         s->n_members_per_group_max = params->n_members_per_group_max;
7835
7836         s->id = p->n_selectors;
7837
7838         /* Node add to tailq. */
7839         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7840         p->n_selectors++;
7841
7842         return 0;
7843
7844 error:
7845         if (!s)
7846                 return status;
7847
7848         free(s->selector_fields);
7849
7850         free(s);
7851
7852         return status;
7853 }
7854
7855 static void
7856 selector_params_free(struct rte_swx_table_selector_params *params)
7857 {
7858         if (!params)
7859                 return;
7860
7861         free(params->selector_mask);
7862
7863         free(params);
7864 }
7865
7866 static struct rte_swx_table_selector_params *
7867 selector_table_params_get(struct selector *s)
7868 {
7869         struct rte_swx_table_selector_params *params = NULL;
7870         struct field *first, *last;
7871         uint32_t i;
7872
7873         /* Memory allocation. */
7874         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7875         if (!params)
7876                 goto error;
7877
7878         /* Group ID. */
7879         params->group_id_offset = s->group_id_field->offset / 8;
7880
7881         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7882         first = s->selector_fields[0];
7883         last = s->selector_fields[0];
7884
7885         for (i = 0; i < s->n_selector_fields; i++) {
7886                 struct field *f = s->selector_fields[i];
7887
7888                 if (f->offset < first->offset)
7889                         first = f;
7890
7891                 if (f->offset > last->offset)
7892                         last = f;
7893         }
7894
7895         /* Selector offset and size. */
7896         params->selector_offset = first->offset / 8;
7897         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7898
7899         /* Memory allocation. */
7900         params->selector_mask = calloc(1, params->selector_size);
7901         if (!params->selector_mask)
7902                 goto error;
7903
7904         /* Selector mask. */
7905         for (i = 0; i < s->n_selector_fields; i++) {
7906                 struct field *f = s->selector_fields[i];
7907                 uint32_t start = (f->offset - first->offset) / 8;
7908                 size_t size = f->n_bits / 8;
7909
7910                 memset(&params->selector_mask[start], 0xFF, size);
7911         }
7912
7913         /* Member ID. */
7914         params->member_id_offset = s->member_id_field->offset / 8;
7915
7916         /* Maximum number of groups. */
7917         params->n_groups_max = s->n_groups_max;
7918
7919         /* Maximum number of members per group. */
7920         params->n_members_per_group_max = s->n_members_per_group_max;
7921
7922         return params;
7923
7924 error:
7925         selector_params_free(params);
7926         return NULL;
7927 }
7928
7929 static void
7930 selector_build_free(struct rte_swx_pipeline *p)
7931 {
7932         uint32_t i;
7933
7934         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7935                 struct thread *t = &p->threads[i];
7936                 uint32_t j;
7937
7938                 if (!t->selectors)
7939                         continue;
7940
7941                 for (j = 0; j < p->n_selectors; j++) {
7942                         struct selector_runtime *r = &t->selectors[j];
7943
7944                         free(r->mailbox);
7945                 }
7946
7947                 free(t->selectors);
7948                 t->selectors = NULL;
7949         }
7950
7951         free(p->selector_stats);
7952         p->selector_stats = NULL;
7953 }
7954
7955 static int
7956 selector_build(struct rte_swx_pipeline *p)
7957 {
7958         uint32_t i;
7959         int status = 0;
7960
7961         /* Per pipeline: selector statistics. */
7962         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7963         if (!p->selector_stats) {
7964                 status = -ENOMEM;
7965                 goto error;
7966         }
7967
7968         /* Per thread: selector run-time. */
7969         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7970                 struct thread *t = &p->threads[i];
7971                 struct selector *s;
7972
7973                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7974                 if (!t->selectors) {
7975                         status = -ENOMEM;
7976                         goto error;
7977                 }
7978
7979                 TAILQ_FOREACH(s, &p->selectors, node) {
7980                         struct selector_runtime *r = &t->selectors[s->id];
7981                         uint64_t size;
7982
7983                         /* r->mailbox. */
7984                         size = rte_swx_table_selector_mailbox_size_get();
7985                         if (size) {
7986                                 r->mailbox = calloc(1, size);
7987                                 if (!r->mailbox) {
7988                                         status = -ENOMEM;
7989                                         goto error;
7990                                 }
7991                         }
7992
7993                         /* r->group_id_buffer. */
7994                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
7995
7996                         /* r->selector_buffer. */
7997                         r->selector_buffer = s->selector_header ?
7998                                 &t->structs[s->selector_header->struct_id] :
7999                                 &t->structs[p->metadata_struct_id];
8000
8001                         /* r->member_id_buffer. */
8002                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8003                 }
8004         }
8005
8006         return 0;
8007
8008 error:
8009         selector_build_free(p);
8010         return status;
8011 }
8012
8013 static void
8014 selector_free(struct rte_swx_pipeline *p)
8015 {
8016         selector_build_free(p);
8017
8018         /* Selector tables. */
8019         for ( ; ; ) {
8020                 struct selector *elem;
8021
8022                 elem = TAILQ_FIRST(&p->selectors);
8023                 if (!elem)
8024                         break;
8025
8026                 TAILQ_REMOVE(&p->selectors, elem, node);
8027                 free(elem->selector_fields);
8028                 free(elem);
8029         }
8030 }
8031
8032 /*
8033  * Learner table.
8034  */
8035 static struct learner *
8036 learner_find(struct rte_swx_pipeline *p, const char *name)
8037 {
8038         struct learner *l;
8039
8040         TAILQ_FOREACH(l, &p->learners, node)
8041                 if (!strcmp(l->name, name))
8042                         return l;
8043
8044         return NULL;
8045 }
8046
8047 static struct learner *
8048 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8049 {
8050         struct learner *l = NULL;
8051
8052         TAILQ_FOREACH(l, &p->learners, node)
8053                 if (l->id == id)
8054                         return l;
8055
8056         return NULL;
8057 }
8058
8059 static int
8060 learner_match_fields_check(struct rte_swx_pipeline *p,
8061                            struct rte_swx_pipeline_learner_params *params,
8062                            struct header **header)
8063 {
8064         struct header *h0 = NULL;
8065         struct field *hf, *mf;
8066         uint32_t i;
8067
8068         /* Return if no match fields. */
8069         if (!params->n_fields || !params->field_names)
8070                 return -EINVAL;
8071
8072         /* Check that all the match fields either belong to the same header
8073          * or are all meta-data fields.
8074          */
8075         hf = header_field_parse(p, params->field_names[0], &h0);
8076         mf = metadata_field_parse(p, params->field_names[0]);
8077         if (!hf && !mf)
8078                 return -EINVAL;
8079
8080         for (i = 1; i < params->n_fields; i++)
8081                 if (h0) {
8082                         struct header *h;
8083
8084                         hf = header_field_parse(p, params->field_names[i], &h);
8085                         if (!hf || (h->id != h0->id))
8086                                 return -EINVAL;
8087                 } else {
8088                         mf = metadata_field_parse(p, params->field_names[i]);
8089                         if (!mf)
8090                                 return -EINVAL;
8091                 }
8092
8093         /* Check that there are no duplicated match fields. */
8094         for (i = 0; i < params->n_fields; i++) {
8095                 const char *field_name = params->field_names[i];
8096                 uint32_t j;
8097
8098                 for (j = i + 1; j < params->n_fields; j++)
8099                         if (!strcmp(params->field_names[j], field_name))
8100                                 return -EINVAL;
8101         }
8102
8103         /* Return. */
8104         if (header)
8105                 *header = h0;
8106
8107         return 0;
8108 }
8109
8110 static int
8111 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8112 {
8113         struct struct_type *mst = p->metadata_st, *ast = a->st;
8114         struct field *mf, *af;
8115         uint32_t mf_pos, i;
8116
8117         if (!ast) {
8118                 if (mf_name)
8119                         return -EINVAL;
8120
8121                 return 0;
8122         }
8123
8124         /* Check that mf_name is the name of a valid meta-data field. */
8125         CHECK_NAME(mf_name, EINVAL);
8126         mf = metadata_field_parse(p, mf_name);
8127         CHECK(mf, EINVAL);
8128
8129         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8130          * all the action arguments.
8131          */
8132         mf_pos = mf - mst->fields;
8133         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8134
8135         /* Check that the size of each of the identified meta-data fields matches exactly the size
8136          * of the corresponding action argument.
8137          */
8138         for (i = 0; i < ast->n_fields; i++) {
8139                 mf = &mst->fields[mf_pos + i];
8140                 af = &ast->fields[i];
8141
8142                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8143         }
8144
8145         return 0;
8146 }
8147
8148 static int
8149 learner_action_learning_check(struct rte_swx_pipeline *p,
8150                               struct action *action,
8151                               const char **action_names,
8152                               uint32_t n_actions)
8153 {
8154         uint32_t i;
8155
8156         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8157          * the action passed as argument to the "learn" instruction) is also enabled for the
8158          * current learner table.
8159          */
8160         for (i = 0; i < action->n_instructions; i++) {
8161                 struct instruction *instr = &action->instructions[i];
8162                 uint32_t found = 0, j;
8163
8164                 if (instr->type != INSTR_LEARNER_LEARN)
8165                         continue;
8166
8167                 for (j = 0; j < n_actions; j++) {
8168                         struct action *a;
8169
8170                         a = action_find(p, action_names[j]);
8171                         if (!a)
8172                                 return -EINVAL;
8173
8174                         if (a->id == instr->learn.action_id)
8175                                 found = 1;
8176                 }
8177
8178                 if (!found)
8179                         return -EINVAL;
8180         }
8181
8182         return 0;
8183 }
8184
8185 int
8186 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8187                               const char *name,
8188                               struct rte_swx_pipeline_learner_params *params,
8189                               uint32_t size,
8190                               uint32_t timeout)
8191 {
8192         struct learner *l = NULL;
8193         struct action *default_action;
8194         struct header *header = NULL;
8195         uint32_t action_data_size_max = 0, i;
8196         int status = 0;
8197
8198         CHECK(p, EINVAL);
8199
8200         CHECK_NAME(name, EINVAL);
8201         CHECK(!table_find(p, name), EEXIST);
8202         CHECK(!selector_find(p, name), EEXIST);
8203         CHECK(!learner_find(p, name), EEXIST);
8204
8205         CHECK(params, EINVAL);
8206
8207         /* Match checks. */
8208         status = learner_match_fields_check(p, params, &header);
8209         if (status)
8210                 return status;
8211
8212         /* Action checks. */
8213         CHECK(params->n_actions, EINVAL);
8214         CHECK(params->action_names, EINVAL);
8215         for (i = 0; i < params->n_actions; i++) {
8216                 const char *action_name = params->action_names[i];
8217                 struct action *a;
8218                 uint32_t action_data_size;
8219                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8220
8221                 CHECK_NAME(action_name, EINVAL);
8222
8223                 a = action_find(p, action_name);
8224                 CHECK(a, EINVAL);
8225
8226                 status = learner_action_learning_check(p,
8227                                                        a,
8228                                                        params->action_names,
8229                                                        params->n_actions);
8230                 if (status)
8231                         return status;
8232
8233                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8234                 if (action_data_size > action_data_size_max)
8235                         action_data_size_max = action_data_size;
8236
8237                 if (params->action_is_for_table_entries)
8238                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8239                 if (params->action_is_for_default_entry)
8240                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8241                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8242         }
8243
8244         CHECK_NAME(params->default_action_name, EINVAL);
8245         for (i = 0; i < p->n_actions; i++)
8246                 if (!strcmp(params->action_names[i],
8247                             params->default_action_name))
8248                         break;
8249         CHECK(i < params->n_actions, EINVAL);
8250         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8251               EINVAL);
8252
8253         default_action = action_find(p, params->default_action_name);
8254         CHECK((default_action->st && params->default_action_data) ||
8255               !params->default_action_data, EINVAL);
8256
8257         /* Any other checks. */
8258         CHECK(size, EINVAL);
8259         CHECK(timeout, EINVAL);
8260
8261         /* Memory allocation. */
8262         l = calloc(1, sizeof(struct learner));
8263         if (!l)
8264                 goto nomem;
8265
8266         l->fields = calloc(params->n_fields, sizeof(struct field *));
8267         if (!l->fields)
8268                 goto nomem;
8269
8270         l->actions = calloc(params->n_actions, sizeof(struct action *));
8271         if (!l->actions)
8272                 goto nomem;
8273
8274         if (action_data_size_max) {
8275                 l->default_action_data = calloc(1, action_data_size_max);
8276                 if (!l->default_action_data)
8277                         goto nomem;
8278         }
8279
8280         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8281         if (!l->action_is_for_table_entries)
8282                 goto nomem;
8283
8284         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8285         if (!l->action_is_for_default_entry)
8286                 goto nomem;
8287
8288         /* Node initialization. */
8289         strcpy(l->name, name);
8290
8291         for (i = 0; i < params->n_fields; i++) {
8292                 const char *field_name = params->field_names[i];
8293
8294                 l->fields[i] = header ?
8295                         header_field_parse(p, field_name, NULL) :
8296                         metadata_field_parse(p, field_name);
8297         }
8298
8299         l->n_fields = params->n_fields;
8300
8301         l->header = header;
8302
8303         for (i = 0; i < params->n_actions; i++) {
8304                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8305
8306                 if (params->action_is_for_table_entries)
8307                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8308                 if (params->action_is_for_default_entry)
8309                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8310
8311                 l->actions[i] = action_find(p, params->action_names[i]);
8312                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8313                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8314         }
8315
8316         l->default_action = default_action;
8317
8318         if (default_action->st)
8319                 memcpy(l->default_action_data,
8320                        params->default_action_data,
8321                        default_action->st->n_bits / 8);
8322
8323         l->n_actions = params->n_actions;
8324
8325         l->default_action_is_const = params->default_action_is_const;
8326
8327         l->action_data_size_max = action_data_size_max;
8328
8329         l->size = size;
8330
8331         l->timeout = timeout;
8332
8333         l->id = p->n_learners;
8334
8335         /* Node add to tailq. */
8336         TAILQ_INSERT_TAIL(&p->learners, l, node);
8337         p->n_learners++;
8338
8339         return 0;
8340
8341 nomem:
8342         if (!l)
8343                 return -ENOMEM;
8344
8345         free(l->action_is_for_default_entry);
8346         free(l->action_is_for_table_entries);
8347         free(l->default_action_data);
8348         free(l->actions);
8349         free(l->fields);
8350         free(l);
8351
8352         return -ENOMEM;
8353 }
8354
8355 static void
8356 learner_params_free(struct rte_swx_table_learner_params *params)
8357 {
8358         if (!params)
8359                 return;
8360
8361         free(params->key_mask0);
8362
8363         free(params);
8364 }
8365
8366 static struct rte_swx_table_learner_params *
8367 learner_params_get(struct learner *l)
8368 {
8369         struct rte_swx_table_learner_params *params = NULL;
8370         struct field *first, *last;
8371         uint32_t i;
8372
8373         /* Memory allocation. */
8374         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8375         if (!params)
8376                 goto error;
8377
8378         /* Find first (smallest offset) and last (biggest offset) match fields. */
8379         first = l->fields[0];
8380         last = l->fields[0];
8381
8382         for (i = 0; i < l->n_fields; i++) {
8383                 struct field *f = l->fields[i];
8384
8385                 if (f->offset < first->offset)
8386                         first = f;
8387
8388                 if (f->offset > last->offset)
8389                         last = f;
8390         }
8391
8392         /* Key offset and size. */
8393         params->key_offset = first->offset / 8;
8394         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8395
8396         /* Memory allocation. */
8397         params->key_mask0 = calloc(1, params->key_size);
8398         if (!params->key_mask0)
8399                 goto error;
8400
8401         /* Key mask. */
8402         for (i = 0; i < l->n_fields; i++) {
8403                 struct field *f = l->fields[i];
8404                 uint32_t start = (f->offset - first->offset) / 8;
8405                 size_t size = f->n_bits / 8;
8406
8407                 memset(&params->key_mask0[start], 0xFF, size);
8408         }
8409
8410         /* Action data size. */
8411         params->action_data_size = l->action_data_size_max;
8412
8413         /* Maximum number of keys. */
8414         params->n_keys_max = l->size;
8415
8416         /* Timeout. */
8417         params->key_timeout = l->timeout;
8418
8419         return params;
8420
8421 error:
8422         learner_params_free(params);
8423         return NULL;
8424 }
8425
8426 static void
8427 learner_build_free(struct rte_swx_pipeline *p)
8428 {
8429         uint32_t i;
8430
8431         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8432                 struct thread *t = &p->threads[i];
8433                 uint32_t j;
8434
8435                 if (!t->learners)
8436                         continue;
8437
8438                 for (j = 0; j < p->n_learners; j++) {
8439                         struct learner_runtime *r = &t->learners[j];
8440
8441                         free(r->mailbox);
8442                 }
8443
8444                 free(t->learners);
8445                 t->learners = NULL;
8446         }
8447
8448         if (p->learner_stats) {
8449                 for (i = 0; i < p->n_learners; i++)
8450                         free(p->learner_stats[i].n_pkts_action);
8451
8452                 free(p->learner_stats);
8453         }
8454 }
8455
8456 static int
8457 learner_build(struct rte_swx_pipeline *p)
8458 {
8459         uint32_t i;
8460         int status = 0;
8461
8462         /* Per pipeline: learner statistics. */
8463         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8464         CHECK(p->learner_stats, ENOMEM);
8465
8466         for (i = 0; i < p->n_learners; i++) {
8467                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8468                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8469         }
8470
8471         /* Per thread: learner run-time. */
8472         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8473                 struct thread *t = &p->threads[i];
8474                 struct learner *l;
8475
8476                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8477                 if (!t->learners) {
8478                         status = -ENOMEM;
8479                         goto error;
8480                 }
8481
8482                 TAILQ_FOREACH(l, &p->learners, node) {
8483                         struct learner_runtime *r = &t->learners[l->id];
8484                         uint64_t size;
8485
8486                         /* r->mailbox. */
8487                         size = rte_swx_table_learner_mailbox_size_get();
8488                         if (size) {
8489                                 r->mailbox = calloc(1, size);
8490                                 if (!r->mailbox) {
8491                                         status = -ENOMEM;
8492                                         goto error;
8493                                 }
8494                         }
8495
8496                         /* r->key. */
8497                         r->key = l->header ?
8498                                 &t->structs[l->header->struct_id] :
8499                                 &t->structs[p->metadata_struct_id];
8500                 }
8501         }
8502
8503         return 0;
8504
8505 error:
8506         learner_build_free(p);
8507         return status;
8508 }
8509
8510 static void
8511 learner_free(struct rte_swx_pipeline *p)
8512 {
8513         learner_build_free(p);
8514
8515         /* Learner tables. */
8516         for ( ; ; ) {
8517                 struct learner *l;
8518
8519                 l = TAILQ_FIRST(&p->learners);
8520                 if (!l)
8521                         break;
8522
8523                 TAILQ_REMOVE(&p->learners, l, node);
8524                 free(l->fields);
8525                 free(l->actions);
8526                 free(l->default_action_data);
8527                 free(l);
8528         }
8529 }
8530
8531 /*
8532  * Table state.
8533  */
8534 static int
8535 table_state_build(struct rte_swx_pipeline *p)
8536 {
8537         struct table *table;
8538         struct selector *s;
8539         struct learner *l;
8540
8541         p->table_state = calloc(p->n_tables + p->n_selectors,
8542                                 sizeof(struct rte_swx_table_state));
8543         CHECK(p->table_state, ENOMEM);
8544
8545         TAILQ_FOREACH(table, &p->tables, node) {
8546                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8547
8548                 if (table->type) {
8549                         struct rte_swx_table_params *params;
8550
8551                         /* ts->obj. */
8552                         params = table_params_get(table);
8553                         CHECK(params, ENOMEM);
8554
8555                         ts->obj = table->type->ops.create(params,
8556                                 NULL,
8557                                 table->args,
8558                                 p->numa_node);
8559
8560                         table_params_free(params);
8561                         CHECK(ts->obj, ENODEV);
8562                 }
8563
8564                 /* ts->default_action_data. */
8565                 if (table->action_data_size_max) {
8566                         ts->default_action_data =
8567                                 malloc(table->action_data_size_max);
8568                         CHECK(ts->default_action_data, ENOMEM);
8569
8570                         memcpy(ts->default_action_data,
8571                                table->default_action_data,
8572                                table->action_data_size_max);
8573                 }
8574
8575                 /* ts->default_action_id. */
8576                 ts->default_action_id = table->default_action->id;
8577         }
8578
8579         TAILQ_FOREACH(s, &p->selectors, node) {
8580                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8581                 struct rte_swx_table_selector_params *params;
8582
8583                 /* ts->obj. */
8584                 params = selector_table_params_get(s);
8585                 CHECK(params, ENOMEM);
8586
8587                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8588
8589                 selector_params_free(params);
8590                 CHECK(ts->obj, ENODEV);
8591         }
8592
8593         TAILQ_FOREACH(l, &p->learners, node) {
8594                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8595                         p->n_selectors + l->id];
8596                 struct rte_swx_table_learner_params *params;
8597
8598                 /* ts->obj. */
8599                 params = learner_params_get(l);
8600                 CHECK(params, ENOMEM);
8601
8602                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8603                 learner_params_free(params);
8604                 CHECK(ts->obj, ENODEV);
8605
8606                 /* ts->default_action_data. */
8607                 if (l->action_data_size_max) {
8608                         ts->default_action_data = malloc(l->action_data_size_max);
8609                         CHECK(ts->default_action_data, ENOMEM);
8610
8611                         memcpy(ts->default_action_data,
8612                                l->default_action_data,
8613                                l->action_data_size_max);
8614                 }
8615
8616                 /* ts->default_action_id. */
8617                 ts->default_action_id = l->default_action->id;
8618         }
8619
8620         return 0;
8621 }
8622
8623 static void
8624 table_state_build_free(struct rte_swx_pipeline *p)
8625 {
8626         uint32_t i;
8627
8628         if (!p->table_state)
8629                 return;
8630
8631         for (i = 0; i < p->n_tables; i++) {
8632                 struct rte_swx_table_state *ts = &p->table_state[i];
8633                 struct table *table = table_find_by_id(p, i);
8634
8635                 /* ts->obj. */
8636                 if (table->type && ts->obj)
8637                         table->type->ops.free(ts->obj);
8638
8639                 /* ts->default_action_data. */
8640                 free(ts->default_action_data);
8641         }
8642
8643         for (i = 0; i < p->n_selectors; i++) {
8644                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8645
8646                 /* ts->obj. */
8647                 if (ts->obj)
8648                         rte_swx_table_selector_free(ts->obj);
8649         }
8650
8651         for (i = 0; i < p->n_learners; i++) {
8652                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8653
8654                 /* ts->obj. */
8655                 if (ts->obj)
8656                         rte_swx_table_learner_free(ts->obj);
8657
8658                 /* ts->default_action_data. */
8659                 free(ts->default_action_data);
8660         }
8661
8662         free(p->table_state);
8663         p->table_state = NULL;
8664 }
8665
8666 static void
8667 table_state_free(struct rte_swx_pipeline *p)
8668 {
8669         table_state_build_free(p);
8670 }
8671
8672 /*
8673  * Register array.
8674  */
8675 static struct regarray *
8676 regarray_find(struct rte_swx_pipeline *p, const char *name)
8677 {
8678         struct regarray *elem;
8679
8680         TAILQ_FOREACH(elem, &p->regarrays, node)
8681                 if (!strcmp(elem->name, name))
8682                         return elem;
8683
8684         return NULL;
8685 }
8686
8687 static struct regarray *
8688 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8689 {
8690         struct regarray *elem = NULL;
8691
8692         TAILQ_FOREACH(elem, &p->regarrays, node)
8693                 if (elem->id == id)
8694                         return elem;
8695
8696         return NULL;
8697 }
8698
8699 int
8700 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8701                               const char *name,
8702                               uint32_t size,
8703                               uint64_t init_val)
8704 {
8705         struct regarray *r;
8706
8707         CHECK(p, EINVAL);
8708
8709         CHECK_NAME(name, EINVAL);
8710         CHECK(!regarray_find(p, name), EEXIST);
8711
8712         CHECK(size, EINVAL);
8713         size = rte_align32pow2(size);
8714
8715         /* Memory allocation. */
8716         r = calloc(1, sizeof(struct regarray));
8717         CHECK(r, ENOMEM);
8718
8719         /* Node initialization. */
8720         strcpy(r->name, name);
8721         r->init_val = init_val;
8722         r->size = size;
8723         r->id = p->n_regarrays;
8724
8725         /* Node add to tailq. */
8726         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8727         p->n_regarrays++;
8728
8729         return 0;
8730 }
8731
8732 static int
8733 regarray_build(struct rte_swx_pipeline *p)
8734 {
8735         struct regarray *regarray;
8736
8737         if (!p->n_regarrays)
8738                 return 0;
8739
8740         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8741         CHECK(p->regarray_runtime, ENOMEM);
8742
8743         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8744                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8745                 uint32_t i;
8746
8747                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8748                                          RTE_CACHE_LINE_SIZE,
8749                                          p->numa_node);
8750                 CHECK(r->regarray, ENOMEM);
8751
8752                 if (regarray->init_val)
8753                         for (i = 0; i < regarray->size; i++)
8754                                 r->regarray[i] = regarray->init_val;
8755
8756                 r->size_mask = regarray->size - 1;
8757         }
8758
8759         return 0;
8760 }
8761
8762 static void
8763 regarray_build_free(struct rte_swx_pipeline *p)
8764 {
8765         uint32_t i;
8766
8767         if (!p->regarray_runtime)
8768                 return;
8769
8770         for (i = 0; i < p->n_regarrays; i++) {
8771                 struct regarray *regarray = regarray_find_by_id(p, i);
8772                 struct regarray_runtime *r = &p->regarray_runtime[i];
8773
8774                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8775         }
8776
8777         free(p->regarray_runtime);
8778         p->regarray_runtime = NULL;
8779 }
8780
8781 static void
8782 regarray_free(struct rte_swx_pipeline *p)
8783 {
8784         regarray_build_free(p);
8785
8786         for ( ; ; ) {
8787                 struct regarray *elem;
8788
8789                 elem = TAILQ_FIRST(&p->regarrays);
8790                 if (!elem)
8791                         break;
8792
8793                 TAILQ_REMOVE(&p->regarrays, elem, node);
8794                 free(elem);
8795         }
8796 }
8797
8798 /*
8799  * Meter array.
8800  */
8801 static struct meter_profile *
8802 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8803 {
8804         struct meter_profile *elem;
8805
8806         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8807                 if (!strcmp(elem->name, name))
8808                         return elem;
8809
8810         return NULL;
8811 }
8812
8813 static struct metarray *
8814 metarray_find(struct rte_swx_pipeline *p, const char *name)
8815 {
8816         struct metarray *elem;
8817
8818         TAILQ_FOREACH(elem, &p->metarrays, node)
8819                 if (!strcmp(elem->name, name))
8820                         return elem;
8821
8822         return NULL;
8823 }
8824
8825 static struct metarray *
8826 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8827 {
8828         struct metarray *elem = NULL;
8829
8830         TAILQ_FOREACH(elem, &p->metarrays, node)
8831                 if (elem->id == id)
8832                         return elem;
8833
8834         return NULL;
8835 }
8836
8837 int
8838 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8839                                  const char *name,
8840                                  uint32_t size)
8841 {
8842         struct metarray *m;
8843
8844         CHECK(p, EINVAL);
8845
8846         CHECK_NAME(name, EINVAL);
8847         CHECK(!metarray_find(p, name), EEXIST);
8848
8849         CHECK(size, EINVAL);
8850         size = rte_align32pow2(size);
8851
8852         /* Memory allocation. */
8853         m = calloc(1, sizeof(struct metarray));
8854         CHECK(m, ENOMEM);
8855
8856         /* Node initialization. */
8857         strcpy(m->name, name);
8858         m->size = size;
8859         m->id = p->n_metarrays;
8860
8861         /* Node add to tailq. */
8862         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8863         p->n_metarrays++;
8864
8865         return 0;
8866 }
8867
8868 struct meter_profile meter_profile_default = {
8869         .node = {0},
8870         .name = "",
8871         .params = {0},
8872
8873         .profile = {
8874                 .cbs = 10000,
8875                 .pbs = 10000,
8876                 .cir_period = 1,
8877                 .cir_bytes_per_period = 1,
8878                 .pir_period = 1,
8879                 .pir_bytes_per_period = 1,
8880         },
8881
8882         .n_users = 0,
8883 };
8884
8885 static void
8886 meter_init(struct meter *m)
8887 {
8888         memset(m, 0, sizeof(struct meter));
8889         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8890         m->profile = &meter_profile_default;
8891         m->color_mask = RTE_COLOR_GREEN;
8892
8893         meter_profile_default.n_users++;
8894 }
8895
8896 static int
8897 metarray_build(struct rte_swx_pipeline *p)
8898 {
8899         struct metarray *m;
8900
8901         if (!p->n_metarrays)
8902                 return 0;
8903
8904         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8905         CHECK(p->metarray_runtime, ENOMEM);
8906
8907         TAILQ_FOREACH(m, &p->metarrays, node) {
8908                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8909                 uint32_t i;
8910
8911                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8912                                          RTE_CACHE_LINE_SIZE,
8913                                          p->numa_node);
8914                 CHECK(r->metarray, ENOMEM);
8915
8916                 for (i = 0; i < m->size; i++)
8917                         meter_init(&r->metarray[i]);
8918
8919                 r->size_mask = m->size - 1;
8920         }
8921
8922         return 0;
8923 }
8924
8925 static void
8926 metarray_build_free(struct rte_swx_pipeline *p)
8927 {
8928         uint32_t i;
8929
8930         if (!p->metarray_runtime)
8931                 return;
8932
8933         for (i = 0; i < p->n_metarrays; i++) {
8934                 struct metarray *m = metarray_find_by_id(p, i);
8935                 struct metarray_runtime *r = &p->metarray_runtime[i];
8936
8937                 env_free(r->metarray, m->size * sizeof(struct meter));
8938         }
8939
8940         free(p->metarray_runtime);
8941         p->metarray_runtime = NULL;
8942 }
8943
8944 static void
8945 metarray_free(struct rte_swx_pipeline *p)
8946 {
8947         metarray_build_free(p);
8948
8949         /* Meter arrays. */
8950         for ( ; ; ) {
8951                 struct metarray *elem;
8952
8953                 elem = TAILQ_FIRST(&p->metarrays);
8954                 if (!elem)
8955                         break;
8956
8957                 TAILQ_REMOVE(&p->metarrays, elem, node);
8958                 free(elem);
8959         }
8960
8961         /* Meter profiles. */
8962         for ( ; ; ) {
8963                 struct meter_profile *elem;
8964
8965                 elem = TAILQ_FIRST(&p->meter_profiles);
8966                 if (!elem)
8967                         break;
8968
8969                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8970                 free(elem);
8971         }
8972 }
8973
8974 /*
8975  * Pipeline.
8976  */
8977 int
8978 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8979 {
8980         struct rte_swx_pipeline *pipeline;
8981
8982         /* Check input parameters. */
8983         CHECK(p, EINVAL);
8984
8985         /* Memory allocation. */
8986         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8987         CHECK(pipeline, ENOMEM);
8988
8989         /* Initialization. */
8990         TAILQ_INIT(&pipeline->struct_types);
8991         TAILQ_INIT(&pipeline->port_in_types);
8992         TAILQ_INIT(&pipeline->ports_in);
8993         TAILQ_INIT(&pipeline->port_out_types);
8994         TAILQ_INIT(&pipeline->ports_out);
8995         TAILQ_INIT(&pipeline->extern_types);
8996         TAILQ_INIT(&pipeline->extern_objs);
8997         TAILQ_INIT(&pipeline->extern_funcs);
8998         TAILQ_INIT(&pipeline->headers);
8999         TAILQ_INIT(&pipeline->actions);
9000         TAILQ_INIT(&pipeline->table_types);
9001         TAILQ_INIT(&pipeline->tables);
9002         TAILQ_INIT(&pipeline->selectors);
9003         TAILQ_INIT(&pipeline->learners);
9004         TAILQ_INIT(&pipeline->regarrays);
9005         TAILQ_INIT(&pipeline->meter_profiles);
9006         TAILQ_INIT(&pipeline->metarrays);
9007
9008         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9009         pipeline->numa_node = numa_node;
9010
9011         *p = pipeline;
9012         return 0;
9013 }
9014
9015 void
9016 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9017 {
9018         void *lib;
9019
9020         if (!p)
9021                 return;
9022
9023         lib = p->lib;
9024
9025         free(p->instruction_data);
9026         free(p->instructions);
9027
9028         metarray_free(p);
9029         regarray_free(p);
9030         table_state_free(p);
9031         learner_free(p);
9032         selector_free(p);
9033         table_free(p);
9034         action_free(p);
9035         instruction_table_free(p);
9036         metadata_free(p);
9037         header_free(p);
9038         extern_func_free(p);
9039         extern_obj_free(p);
9040         port_out_free(p);
9041         port_in_free(p);
9042         struct_free(p);
9043
9044         free(p);
9045
9046         if (lib)
9047                 dlclose(lib);
9048 }
9049
9050 int
9051 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9052                                      const char **instructions,
9053                                      uint32_t n_instructions)
9054 {
9055         int err;
9056         uint32_t i;
9057
9058         err = instruction_config(p, NULL, instructions, n_instructions);
9059         if (err)
9060                 return err;
9061
9062         /* Thread instruction pointer reset. */
9063         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9064                 struct thread *t = &p->threads[i];
9065
9066                 thread_ip_reset(p, t);
9067         }
9068
9069         return 0;
9070 }
9071
9072 static int
9073 pipeline_compile(struct rte_swx_pipeline *p);
9074
9075 int
9076 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9077 {
9078         int status;
9079
9080         CHECK(p, EINVAL);
9081         CHECK(p->build_done == 0, EEXIST);
9082
9083         status = port_in_build(p);
9084         if (status)
9085                 goto error;
9086
9087         status = port_out_build(p);
9088         if (status)
9089                 goto error;
9090
9091         status = struct_build(p);
9092         if (status)
9093                 goto error;
9094
9095         status = extern_obj_build(p);
9096         if (status)
9097                 goto error;
9098
9099         status = extern_func_build(p);
9100         if (status)
9101                 goto error;
9102
9103         status = header_build(p);
9104         if (status)
9105                 goto error;
9106
9107         status = metadata_build(p);
9108         if (status)
9109                 goto error;
9110
9111         status = instruction_table_build(p);
9112         if (status)
9113                 goto error;
9114
9115         status = action_build(p);
9116         if (status)
9117                 goto error;
9118
9119         status = table_build(p);
9120         if (status)
9121                 goto error;
9122
9123         status = selector_build(p);
9124         if (status)
9125                 goto error;
9126
9127         status = learner_build(p);
9128         if (status)
9129                 goto error;
9130
9131         status = table_state_build(p);
9132         if (status)
9133                 goto error;
9134
9135         status = regarray_build(p);
9136         if (status)
9137                 goto error;
9138
9139         status = metarray_build(p);
9140         if (status)
9141                 goto error;
9142
9143         p->build_done = 1;
9144
9145         pipeline_compile(p);
9146
9147         return 0;
9148
9149 error:
9150         metarray_build_free(p);
9151         regarray_build_free(p);
9152         table_state_build_free(p);
9153         learner_build_free(p);
9154         selector_build_free(p);
9155         table_build_free(p);
9156         action_build_free(p);
9157         instruction_table_build_free(p);
9158         metadata_build_free(p);
9159         header_build_free(p);
9160         extern_func_build_free(p);
9161         extern_obj_build_free(p);
9162         port_out_build_free(p);
9163         port_in_build_free(p);
9164         struct_build_free(p);
9165
9166         return status;
9167 }
9168
9169 void
9170 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9171 {
9172         uint32_t i;
9173
9174         for (i = 0; i < n_instructions; i++)
9175                 instr_exec(p);
9176 }
9177
9178 void
9179 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9180 {
9181         uint32_t i;
9182
9183         for (i = 0; i < p->n_ports_out; i++) {
9184                 struct port_out_runtime *port = &p->out[i];
9185
9186                 if (port->flush)
9187                         port->flush(port->obj);
9188         }
9189 }
9190
9191 /*
9192  * Control.
9193  */
9194 int
9195 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9196                               struct rte_swx_ctl_pipeline_info *pipeline)
9197 {
9198         struct action *action;
9199         struct table *table;
9200         uint32_t n_actions = 0, n_tables = 0;
9201
9202         if (!p || !pipeline)
9203                 return -EINVAL;
9204
9205         TAILQ_FOREACH(action, &p->actions, node)
9206                 n_actions++;
9207
9208         TAILQ_FOREACH(table, &p->tables, node)
9209                 n_tables++;
9210
9211         pipeline->n_ports_in = p->n_ports_in;
9212         pipeline->n_ports_out = p->n_ports_out;
9213         pipeline->n_actions = n_actions;
9214         pipeline->n_tables = n_tables;
9215         pipeline->n_selectors = p->n_selectors;
9216         pipeline->n_learners = p->n_learners;
9217         pipeline->n_regarrays = p->n_regarrays;
9218         pipeline->n_metarrays = p->n_metarrays;
9219
9220         return 0;
9221 }
9222
9223 int
9224 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9225 {
9226         if (!p || !numa_node)
9227                 return -EINVAL;
9228
9229         *numa_node = p->numa_node;
9230         return 0;
9231 }
9232
9233 int
9234 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9235                             uint32_t action_id,
9236                             struct rte_swx_ctl_action_info *action)
9237 {
9238         struct action *a = NULL;
9239
9240         if (!p || (action_id >= p->n_actions) || !action)
9241                 return -EINVAL;
9242
9243         a = action_find_by_id(p, action_id);
9244         if (!a)
9245                 return -EINVAL;
9246
9247         strcpy(action->name, a->name);
9248         action->n_args = a->st ? a->st->n_fields : 0;
9249         return 0;
9250 }
9251
9252 int
9253 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9254                                 uint32_t action_id,
9255                                 uint32_t action_arg_id,
9256                                 struct rte_swx_ctl_action_arg_info *action_arg)
9257 {
9258         struct action *a = NULL;
9259         struct field *arg = NULL;
9260
9261         if (!p || (action_id >= p->n_actions) || !action_arg)
9262                 return -EINVAL;
9263
9264         a = action_find_by_id(p, action_id);
9265         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9266                 return -EINVAL;
9267
9268         arg = &a->st->fields[action_arg_id];
9269         strcpy(action_arg->name, arg->name);
9270         action_arg->n_bits = arg->n_bits;
9271         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9272
9273         return 0;
9274 }
9275
9276 int
9277 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9278                            uint32_t table_id,
9279                            struct rte_swx_ctl_table_info *table)
9280 {
9281         struct table *t = NULL;
9282
9283         if (!p || !table)
9284                 return -EINVAL;
9285
9286         t = table_find_by_id(p, table_id);
9287         if (!t)
9288                 return -EINVAL;
9289
9290         strcpy(table->name, t->name);
9291         strcpy(table->args, t->args);
9292         table->n_match_fields = t->n_fields;
9293         table->n_actions = t->n_actions;
9294         table->default_action_is_const = t->default_action_is_const;
9295         table->size = t->size;
9296         return 0;
9297 }
9298
9299 int
9300 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9301         uint32_t table_id,
9302         uint32_t match_field_id,
9303         struct rte_swx_ctl_table_match_field_info *match_field)
9304 {
9305         struct table *t;
9306         struct match_field *f;
9307
9308         if (!p || (table_id >= p->n_tables) || !match_field)
9309                 return -EINVAL;
9310
9311         t = table_find_by_id(p, table_id);
9312         if (!t || (match_field_id >= t->n_fields))
9313                 return -EINVAL;
9314
9315         f = &t->fields[match_field_id];
9316         match_field->match_type = f->match_type;
9317         match_field->is_header = t->header ? 1 : 0;
9318         match_field->n_bits = f->field->n_bits;
9319         match_field->offset = f->field->offset;
9320
9321         return 0;
9322 }
9323
9324 int
9325 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9326         uint32_t table_id,
9327         uint32_t table_action_id,
9328         struct rte_swx_ctl_table_action_info *table_action)
9329 {
9330         struct table *t;
9331
9332         if (!p || (table_id >= p->n_tables) || !table_action)
9333                 return -EINVAL;
9334
9335         t = table_find_by_id(p, table_id);
9336         if (!t || (table_action_id >= t->n_actions))
9337                 return -EINVAL;
9338
9339         table_action->action_id = t->actions[table_action_id]->id;
9340
9341         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9342         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9343
9344         return 0;
9345 }
9346
9347 int
9348 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9349                           uint32_t table_id,
9350                           struct rte_swx_table_ops *table_ops,
9351                           int *is_stub)
9352 {
9353         struct table *t;
9354
9355         if (!p || (table_id >= p->n_tables))
9356                 return -EINVAL;
9357
9358         t = table_find_by_id(p, table_id);
9359         if (!t)
9360                 return -EINVAL;
9361
9362         if (t->type) {
9363                 if (table_ops)
9364                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9365                 *is_stub = 0;
9366         } else {
9367                 *is_stub = 1;
9368         }
9369
9370         return 0;
9371 }
9372
9373 int
9374 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9375                               uint32_t selector_id,
9376                               struct rte_swx_ctl_selector_info *selector)
9377 {
9378         struct selector *s = NULL;
9379
9380         if (!p || !selector)
9381                 return -EINVAL;
9382
9383         s = selector_find_by_id(p, selector_id);
9384         if (!s)
9385                 return -EINVAL;
9386
9387         strcpy(selector->name, s->name);
9388
9389         selector->n_selector_fields = s->n_selector_fields;
9390         selector->n_groups_max = s->n_groups_max;
9391         selector->n_members_per_group_max = s->n_members_per_group_max;
9392
9393         return 0;
9394 }
9395
9396 int
9397 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9398          uint32_t selector_id,
9399          struct rte_swx_ctl_table_match_field_info *field)
9400 {
9401         struct selector *s;
9402
9403         if (!p || (selector_id >= p->n_selectors) || !field)
9404                 return -EINVAL;
9405
9406         s = selector_find_by_id(p, selector_id);
9407         if (!s)
9408                 return -EINVAL;
9409
9410         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9411         field->is_header = 0;
9412         field->n_bits = s->group_id_field->n_bits;
9413         field->offset = s->group_id_field->offset;
9414
9415         return 0;
9416 }
9417
9418 int
9419 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9420          uint32_t selector_id,
9421          uint32_t selector_field_id,
9422          struct rte_swx_ctl_table_match_field_info *field)
9423 {
9424         struct selector *s;
9425         struct field *f;
9426
9427         if (!p || (selector_id >= p->n_selectors) || !field)
9428                 return -EINVAL;
9429
9430         s = selector_find_by_id(p, selector_id);
9431         if (!s || (selector_field_id >= s->n_selector_fields))
9432                 return -EINVAL;
9433
9434         f = s->selector_fields[selector_field_id];
9435         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9436         field->is_header = s->selector_header ? 1 : 0;
9437         field->n_bits = f->n_bits;
9438         field->offset = f->offset;
9439
9440         return 0;
9441 }
9442
9443 int
9444 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9445          uint32_t selector_id,
9446          struct rte_swx_ctl_table_match_field_info *field)
9447 {
9448         struct selector *s;
9449
9450         if (!p || (selector_id >= p->n_selectors) || !field)
9451                 return -EINVAL;
9452
9453         s = selector_find_by_id(p, selector_id);
9454         if (!s)
9455                 return -EINVAL;
9456
9457         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9458         field->is_header = 0;
9459         field->n_bits = s->member_id_field->n_bits;
9460         field->offset = s->member_id_field->offset;
9461
9462         return 0;
9463 }
9464
9465 int
9466 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9467                              uint32_t learner_id,
9468                              struct rte_swx_ctl_learner_info *learner)
9469 {
9470         struct learner *l = NULL;
9471
9472         if (!p || !learner)
9473                 return -EINVAL;
9474
9475         l = learner_find_by_id(p, learner_id);
9476         if (!l)
9477                 return -EINVAL;
9478
9479         strcpy(learner->name, l->name);
9480
9481         learner->n_match_fields = l->n_fields;
9482         learner->n_actions = l->n_actions;
9483         learner->default_action_is_const = l->default_action_is_const;
9484         learner->size = l->size;
9485
9486         return 0;
9487 }
9488
9489 int
9490 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9491                                          uint32_t learner_id,
9492                                          uint32_t match_field_id,
9493                                          struct rte_swx_ctl_table_match_field_info *match_field)
9494 {
9495         struct learner *l;
9496         struct field *f;
9497
9498         if (!p || (learner_id >= p->n_learners) || !match_field)
9499                 return -EINVAL;
9500
9501         l = learner_find_by_id(p, learner_id);
9502         if (!l || (match_field_id >= l->n_fields))
9503                 return -EINVAL;
9504
9505         f = l->fields[match_field_id];
9506         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9507         match_field->is_header = l->header ? 1 : 0;
9508         match_field->n_bits = f->n_bits;
9509         match_field->offset = f->offset;
9510
9511         return 0;
9512 }
9513
9514 int
9515 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9516                                     uint32_t learner_id,
9517                                     uint32_t learner_action_id,
9518                                     struct rte_swx_ctl_table_action_info *learner_action)
9519 {
9520         struct learner *l;
9521
9522         if (!p || (learner_id >= p->n_learners) || !learner_action)
9523                 return -EINVAL;
9524
9525         l = learner_find_by_id(p, learner_id);
9526         if (!l || (learner_action_id >= l->n_actions))
9527                 return -EINVAL;
9528
9529         learner_action->action_id = l->actions[learner_action_id]->id;
9530
9531         learner_action->action_is_for_table_entries =
9532                 l->action_is_for_table_entries[learner_action_id];
9533
9534         learner_action->action_is_for_default_entry =
9535                 l->action_is_for_default_entry[learner_action_id];
9536
9537         return 0;
9538 }
9539
9540 int
9541 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9542                                  struct rte_swx_table_state **table_state)
9543 {
9544         if (!p || !table_state || !p->build_done)
9545                 return -EINVAL;
9546
9547         *table_state = p->table_state;
9548         return 0;
9549 }
9550
9551 int
9552 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9553                                  struct rte_swx_table_state *table_state)
9554 {
9555         if (!p || !table_state || !p->build_done)
9556                 return -EINVAL;
9557
9558         p->table_state = table_state;
9559         return 0;
9560 }
9561
9562 int
9563 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9564                                         uint32_t port_id,
9565                                         struct rte_swx_port_in_stats *stats)
9566 {
9567         struct port_in *port;
9568
9569         if (!p || !stats)
9570                 return -EINVAL;
9571
9572         port = port_in_find(p, port_id);
9573         if (!port)
9574                 return -EINVAL;
9575
9576         port->type->ops.stats_read(port->obj, stats);
9577         return 0;
9578 }
9579
9580 int
9581 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9582                                          uint32_t port_id,
9583                                          struct rte_swx_port_out_stats *stats)
9584 {
9585         struct port_out *port;
9586
9587         if (!p || !stats)
9588                 return -EINVAL;
9589
9590         port = port_out_find(p, port_id);
9591         if (!port)
9592                 return -EINVAL;
9593
9594         port->type->ops.stats_read(port->obj, stats);
9595         return 0;
9596 }
9597
9598 int
9599 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9600                                       const char *table_name,
9601                                       struct rte_swx_table_stats *stats)
9602 {
9603         struct table *table;
9604         struct table_statistics *table_stats;
9605
9606         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9607                 return -EINVAL;
9608
9609         table = table_find(p, table_name);
9610         if (!table)
9611                 return -EINVAL;
9612
9613         table_stats = &p->table_stats[table->id];
9614
9615         memcpy(stats->n_pkts_action,
9616                table_stats->n_pkts_action,
9617                p->n_actions * sizeof(uint64_t));
9618
9619         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9620         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9621
9622         return 0;
9623 }
9624
9625 int
9626 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9627         const char *selector_name,
9628         struct rte_swx_pipeline_selector_stats *stats)
9629 {
9630         struct selector *s;
9631
9632         if (!p || !selector_name || !selector_name[0] || !stats)
9633                 return -EINVAL;
9634
9635         s = selector_find(p, selector_name);
9636         if (!s)
9637                 return -EINVAL;
9638
9639         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9640
9641         return 0;
9642 }
9643
9644 int
9645 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9646                                         const char *learner_name,
9647                                         struct rte_swx_learner_stats *stats)
9648 {
9649         struct learner *l;
9650         struct learner_statistics *learner_stats;
9651
9652         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9653                 return -EINVAL;
9654
9655         l = learner_find(p, learner_name);
9656         if (!l)
9657                 return -EINVAL;
9658
9659         learner_stats = &p->learner_stats[l->id];
9660
9661         memcpy(stats->n_pkts_action,
9662                learner_stats->n_pkts_action,
9663                p->n_actions * sizeof(uint64_t));
9664
9665         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9666         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9667
9668         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9669         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9670
9671         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9672
9673         return 0;
9674 }
9675
9676 int
9677 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9678                               uint32_t regarray_id,
9679                               struct rte_swx_ctl_regarray_info *regarray)
9680 {
9681         struct regarray *r;
9682
9683         if (!p || !regarray)
9684                 return -EINVAL;
9685
9686         r = regarray_find_by_id(p, regarray_id);
9687         if (!r)
9688                 return -EINVAL;
9689
9690         strcpy(regarray->name, r->name);
9691         regarray->size = r->size;
9692         return 0;
9693 }
9694
9695 int
9696 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9697                                    const char *regarray_name,
9698                                    uint32_t regarray_index,
9699                                    uint64_t *value)
9700 {
9701         struct regarray *regarray;
9702         struct regarray_runtime *r;
9703
9704         if (!p || !regarray_name || !value)
9705                 return -EINVAL;
9706
9707         regarray = regarray_find(p, regarray_name);
9708         if (!regarray || (regarray_index >= regarray->size))
9709                 return -EINVAL;
9710
9711         r = &p->regarray_runtime[regarray->id];
9712         *value = r->regarray[regarray_index];
9713         return 0;
9714 }
9715
9716 int
9717 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9718                                    const char *regarray_name,
9719                                    uint32_t regarray_index,
9720                                    uint64_t value)
9721 {
9722         struct regarray *regarray;
9723         struct regarray_runtime *r;
9724
9725         if (!p || !regarray_name)
9726                 return -EINVAL;
9727
9728         regarray = regarray_find(p, regarray_name);
9729         if (!regarray || (regarray_index >= regarray->size))
9730                 return -EINVAL;
9731
9732         r = &p->regarray_runtime[regarray->id];
9733         r->regarray[regarray_index] = value;
9734         return 0;
9735 }
9736
9737 int
9738 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9739                               uint32_t metarray_id,
9740                               struct rte_swx_ctl_metarray_info *metarray)
9741 {
9742         struct metarray *m;
9743
9744         if (!p || !metarray)
9745                 return -EINVAL;
9746
9747         m = metarray_find_by_id(p, metarray_id);
9748         if (!m)
9749                 return -EINVAL;
9750
9751         strcpy(metarray->name, m->name);
9752         metarray->size = m->size;
9753         return 0;
9754 }
9755
9756 int
9757 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9758                               const char *name,
9759                               struct rte_meter_trtcm_params *params)
9760 {
9761         struct meter_profile *mp;
9762         int status;
9763
9764         CHECK(p, EINVAL);
9765         CHECK_NAME(name, EINVAL);
9766         CHECK(params, EINVAL);
9767         CHECK(!meter_profile_find(p, name), EEXIST);
9768
9769         /* Node allocation. */
9770         mp = calloc(1, sizeof(struct meter_profile));
9771         CHECK(mp, ENOMEM);
9772
9773         /* Node initialization. */
9774         strcpy(mp->name, name);
9775         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9776         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9777         if (status) {
9778                 free(mp);
9779                 CHECK(0, EINVAL);
9780         }
9781
9782         /* Node add to tailq. */
9783         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9784
9785         return 0;
9786 }
9787
9788 int
9789 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9790                                  const char *name)
9791 {
9792         struct meter_profile *mp;
9793
9794         CHECK(p, EINVAL);
9795         CHECK_NAME(name, EINVAL);
9796
9797         mp = meter_profile_find(p, name);
9798         CHECK(mp, EINVAL);
9799         CHECK(!mp->n_users, EBUSY);
9800
9801         /* Remove node from tailq. */
9802         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9803         free(mp);
9804
9805         return 0;
9806 }
9807
9808 int
9809 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9810                         const char *metarray_name,
9811                         uint32_t metarray_index)
9812 {
9813         struct meter_profile *mp_old;
9814         struct metarray *metarray;
9815         struct metarray_runtime *metarray_runtime;
9816         struct meter *m;
9817
9818         CHECK(p, EINVAL);
9819         CHECK_NAME(metarray_name, EINVAL);
9820
9821         metarray = metarray_find(p, metarray_name);
9822         CHECK(metarray, EINVAL);
9823         CHECK(metarray_index < metarray->size, EINVAL);
9824
9825         metarray_runtime = &p->metarray_runtime[metarray->id];
9826         m = &metarray_runtime->metarray[metarray_index];
9827         mp_old = m->profile;
9828
9829         meter_init(m);
9830
9831         mp_old->n_users--;
9832
9833         return 0;
9834 }
9835
9836 int
9837 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9838                       const char *metarray_name,
9839                       uint32_t metarray_index,
9840                       const char *profile_name)
9841 {
9842         struct meter_profile *mp, *mp_old;
9843         struct metarray *metarray;
9844         struct metarray_runtime *metarray_runtime;
9845         struct meter *m;
9846
9847         CHECK(p, EINVAL);
9848         CHECK_NAME(metarray_name, EINVAL);
9849
9850         metarray = metarray_find(p, metarray_name);
9851         CHECK(metarray, EINVAL);
9852         CHECK(metarray_index < metarray->size, EINVAL);
9853
9854         mp = meter_profile_find(p, profile_name);
9855         CHECK(mp, EINVAL);
9856
9857         metarray_runtime = &p->metarray_runtime[metarray->id];
9858         m = &metarray_runtime->metarray[metarray_index];
9859         mp_old = m->profile;
9860
9861         memset(m, 0, sizeof(struct meter));
9862         rte_meter_trtcm_config(&m->m, &mp->profile);
9863         m->profile = mp;
9864         m->color_mask = RTE_COLORS;
9865
9866         mp->n_users++;
9867         mp_old->n_users--;
9868
9869         return 0;
9870 }
9871
9872 int
9873 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9874                              const char *metarray_name,
9875                              uint32_t metarray_index,
9876                              struct rte_swx_ctl_meter_stats *stats)
9877 {
9878         struct metarray *metarray;
9879         struct metarray_runtime *metarray_runtime;
9880         struct meter *m;
9881
9882         CHECK(p, EINVAL);
9883         CHECK_NAME(metarray_name, EINVAL);
9884
9885         metarray = metarray_find(p, metarray_name);
9886         CHECK(metarray, EINVAL);
9887         CHECK(metarray_index < metarray->size, EINVAL);
9888
9889         CHECK(stats, EINVAL);
9890
9891         metarray_runtime = &p->metarray_runtime[metarray->id];
9892         m = &metarray_runtime->metarray[metarray_index];
9893
9894         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9895         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9896
9897         return 0;
9898 }
9899
9900 /*
9901  * Pipeline compilation.
9902  */
9903 static const char *
9904 instr_type_to_name(struct instruction *instr)
9905 {
9906         switch (instr->type) {
9907         case INSTR_RX: return "INSTR_RX";
9908
9909         case INSTR_TX: return "INSTR_TX";
9910         case INSTR_TX_I: return "INSTR_TX_I";
9911
9912         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
9913         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
9914         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
9915         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
9916         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
9917         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
9918         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
9919         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
9920
9921         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
9922
9923         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
9924
9925         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
9926         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
9927         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
9928         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
9929         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
9930         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
9931         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
9932         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
9933         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
9934
9935         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
9936         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
9937
9938         case INSTR_MOV: return "INSTR_MOV";
9939         case INSTR_MOV_MH: return "INSTR_MOV_MH";
9940         case INSTR_MOV_HM: return "INSTR_MOV_HM";
9941         case INSTR_MOV_HH: return "INSTR_MOV_HH";
9942         case INSTR_MOV_I: return "INSTR_MOV_I";
9943
9944         case INSTR_DMA_HT: return "INSTR_DMA_HT";
9945         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
9946         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
9947         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
9948         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
9949         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
9950         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
9951         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
9952
9953         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
9954         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
9955         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
9956         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
9957         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
9958         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
9959
9960         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
9961         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
9962         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
9963         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
9964         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
9965         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
9966
9967         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
9968         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
9969         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
9970         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
9971
9972         case INSTR_ALU_AND: return "INSTR_ALU_AND";
9973         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
9974         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
9975         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
9976         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
9977
9978         case INSTR_ALU_OR: return "INSTR_ALU_OR";
9979         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
9980         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
9981         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
9982         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
9983
9984         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
9985         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
9986         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
9987         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
9988         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
9989
9990         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
9991         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
9992         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
9993         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
9994         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
9995         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
9996
9997         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
9998         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
9999         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10000         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10001         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10002         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10003
10004         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10005         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10006         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10007
10008         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10009         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10010         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10011         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10012         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10013         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10014
10015         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10016         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10017         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10018         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10019         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10020         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10021         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10022         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10023         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10024
10025         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10026         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10027         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10028         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10029         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10030         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10031         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10032         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10033         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10034
10035         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10036         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10037         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10038
10039         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10040         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10041         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10042         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10043         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10044         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10045         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10046         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10047         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10048         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10049         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10050         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10051
10052         case INSTR_TABLE: return "INSTR_TABLE";
10053         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10054         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10055         case INSTR_LEARNER: return "INSTR_LEARNER";
10056         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10057
10058         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10059         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10060
10061         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10062         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10063
10064         case INSTR_JMP: return "INSTR_JMP";
10065         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10066         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10067         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10068         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10069         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10070         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10071         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10072         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10073         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10074         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10075         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10076         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10077         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10078         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10079         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10080         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10081         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10082         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10083         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10084         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10085         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10086         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10087         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10088         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10089         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10090         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10091         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10092         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10093
10094         case INSTR_RETURN: return "INSTR_RETURN";
10095
10096         default: return "INSTR_UNKNOWN";
10097         }
10098 }
10099
10100 typedef void
10101 (*instruction_export_t)(struct instruction *, FILE *);
10102
10103 static void
10104 instr_io_export(struct instruction *instr, FILE *f)
10105 {
10106         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10107
10108         /* n_io, n_io_imm, n_hdrs. */
10109         if (instr->type == INSTR_RX ||
10110             instr->type == INSTR_TX ||
10111             instr->type == INSTR_HDR_EXTRACT_M ||
10112             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10113                 n_io = 1;
10114
10115         if (instr->type == INSTR_TX_I)
10116                 n_io_imm = 1;
10117
10118         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10119                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10120
10121         if (instr->type == INSTR_HDR_EXTRACT_M ||
10122             instr->type == INSTR_HDR_LOOKAHEAD ||
10123             instr->type == INSTR_HDR_EMIT)
10124                 n_hdrs = 1;
10125
10126         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10127                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10128
10129         /* instr. */
10130         fprintf(f,
10131                 "\t{\n"
10132                 "\t\t.type = %s,\n",
10133                 instr_type_to_name(instr));
10134
10135         /* instr.io. */
10136         fprintf(f,
10137                 "\t\t.io = {\n");
10138
10139         /* instr.io.io. */
10140         if (n_io)
10141                 fprintf(f,
10142                         "\t\t\t.io = {\n"
10143                         "\t\t\t\t.offset = %u,\n"
10144                         "\t\t\t\t.n_bits = %u,\n"
10145                         "\t\t\t},\n",
10146                         instr->io.io.offset,
10147                         instr->io.io.n_bits);
10148
10149         if (n_io_imm)
10150                 fprintf(f,
10151                         "\t\t\t.io = {\n"
10152                         "\t\t\t\t.val = %u,\n"
10153                         "\t\t\t},\n",
10154                         instr->io.io.val);
10155
10156         /* instr.io.hdr. */
10157         if (n_hdrs) {
10158                 fprintf(f,
10159                         "\t\t.hdr = {\n");
10160
10161                 /* instr.io.hdr.header_id. */
10162                 fprintf(f,
10163                         "\t\t\t.header_id = {");
10164
10165                 for (i = 0; i < n_hdrs; i++)
10166                         fprintf(f,
10167                                 "%u, ",
10168                                 instr->io.hdr.header_id[i]);
10169
10170                 fprintf(f,
10171                         "},\n");
10172
10173                 /* instr.io.hdr.struct_id. */
10174                 fprintf(f,
10175                         "\t\t\t.struct_id = {");
10176
10177                 for (i = 0; i < n_hdrs; i++)
10178                         fprintf(f,
10179                                 "%u, ",
10180                                 instr->io.hdr.struct_id[i]);
10181
10182                 fprintf(f,
10183                         "},\n");
10184
10185                 /* instr.io.hdr.n_bytes. */
10186                 fprintf(f,
10187                         "\t\t\t.n_bytes = {");
10188
10189                 for (i = 0; i < n_hdrs; i++)
10190                         fprintf(f,
10191                                 "%u, ",
10192                                 instr->io.hdr.n_bytes[i]);
10193
10194                 fprintf(f,
10195                         "},\n");
10196
10197                 /* instr.io.hdr - closing curly brace. */
10198                 fprintf(f,
10199                         "\t\t\t}\n,");
10200         }
10201
10202         /* instr.io - closing curly brace. */
10203         fprintf(f,
10204                 "\t\t},\n");
10205
10206         /* instr - closing curly brace. */
10207         fprintf(f,
10208                 "\t},\n");
10209 }
10210
10211 static void
10212 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10213 {
10214         fprintf(f,
10215                 "\t{\n"
10216                 "\t\t.type = %s,\n"
10217                 "\t\t.valid = {\n"
10218                 "\t\t\t.header_id = %u,\n"
10219                 "\t\t},\n"
10220                 "\t},\n",
10221                 instr_type_to_name(instr),
10222                 instr->valid.header_id);
10223 }
10224
10225 static void
10226 instr_mov_export(struct instruction *instr, FILE *f)
10227 {
10228         if (instr->type != INSTR_MOV_I)
10229                 fprintf(f,
10230                         "\t{\n"
10231                         "\t\t.type = %s,\n"
10232                         "\t\t.mov = {\n"
10233                         "\t\t\t.dst = {\n"
10234                         "\t\t\t\t.struct_id = %u,\n"
10235                         "\t\t\t\t.n_bits = %u,\n"
10236                         "\t\t\t\t.offset = %u,\n"
10237                         "\t\t\t},\n"
10238                         "\t\t\t.src = {\n"
10239                         "\t\t\t\t.struct_id = %u,\n"
10240                         "\t\t\t\t.n_bits = %u,\n"
10241                         "\t\t\t\t.offset = %u,\n"
10242                         "\t\t\t},\n"
10243                         "\t\t},\n"
10244                         "\t},\n",
10245                         instr_type_to_name(instr),
10246                         instr->mov.dst.struct_id,
10247                         instr->mov.dst.n_bits,
10248                         instr->mov.dst.offset,
10249                         instr->mov.src.struct_id,
10250                         instr->mov.src.n_bits,
10251                         instr->mov.src.offset);
10252         else
10253                 fprintf(f,
10254                         "\t{\n"
10255                         "\t\t.type = %s,\n"
10256                         "\t\t.mov = {\n"
10257                         "\t\t\t.dst = {\n"
10258                         "\t\t\t\t.struct_id = %u,\n"
10259                         "\t\t\t\t.n_bits = %u,\n"
10260                         "\t\t\t\t.offset = %u,\n"
10261                         "\t\t\t}\n,"
10262                         "\t\t\t.src_val = %" PRIu64 ",\n"
10263                         "\t\t},\n"
10264                         "\t},\n",
10265                         instr_type_to_name(instr),
10266                         instr->mov.dst.struct_id,
10267                         instr->mov.dst.n_bits,
10268                         instr->mov.dst.offset,
10269                         instr->mov.src_val);
10270 }
10271
10272 static void
10273 instr_dma_ht_export(struct instruction *instr, FILE *f)
10274 {
10275         uint32_t n_dma = 0, i;
10276
10277         /* n_dma. */
10278         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10279
10280         /* instr. */
10281         fprintf(f,
10282                 "\t{\n"
10283                 "\t\t.type = %s,\n",
10284                 instr_type_to_name(instr));
10285
10286         /* instr.dma. */
10287         fprintf(f,
10288                 "\t\t.dma = {\n");
10289
10290         /* instr.dma.dst. */
10291         fprintf(f,
10292                 "\t\t\t.dst = {\n");
10293
10294         /* instr.dma.dst.header_id. */
10295         fprintf(f,
10296                 "\t\t\t\t.header_id = {");
10297
10298         for (i = 0; i < n_dma; i++)
10299                 fprintf(f,
10300                         "%u, ",
10301                         instr->dma.dst.header_id[i]);
10302
10303         fprintf(f,
10304                 "},\n");
10305
10306         /* instr.dma.dst.struct_id. */
10307         fprintf(f,
10308                 "\t\t\t\t.struct_id = {");
10309
10310         for (i = 0; i < n_dma; i++)
10311                 fprintf(f,
10312                         "%u, ",
10313                         instr->dma.dst.struct_id[i]);
10314
10315         fprintf(f,
10316                 "},\n");
10317
10318         /* instr.dma.dst - closing curly brace. */
10319         fprintf(f,
10320                 "\t\t\t},\n");
10321
10322         /* instr.dma.src. */
10323         fprintf(f,
10324                 "\t\t\t.src = {\n");
10325
10326         /* instr.dma.src.offset. */
10327         fprintf(f,
10328                 "\t\t\t\t.offset = {");
10329
10330         for (i = 0; i < n_dma; i++)
10331                 fprintf(f,
10332                         "%u, ",
10333                         instr->dma.src.offset[i]);
10334
10335         fprintf(f,
10336                 "},\n");
10337
10338         /* instr.dma.src - closing curly brace. */
10339         fprintf(f,
10340                 "\t\t\t},\n");
10341
10342         /* instr.dma.n_bytes. */
10343         fprintf(f,
10344                 "\t\t\t.n_bytes = {");
10345
10346         for (i = 0; i < n_dma; i++)
10347                 fprintf(f,
10348                         "%u, ",
10349                         instr->dma.n_bytes[i]);
10350
10351         fprintf(f,
10352                 "},\n");
10353
10354         /* instr.dma - closing curly brace. */
10355         fprintf(f,
10356                 "\t\t},\n");
10357
10358         /* instr - closing curly brace. */
10359         fprintf(f,
10360                 "\t},\n");
10361 }
10362
10363 static void
10364 instr_alu_export(struct instruction *instr, FILE *f)
10365 {
10366         int imm = 0;
10367
10368         if (instr->type == INSTR_ALU_ADD_MI ||
10369             instr->type == INSTR_ALU_ADD_HI ||
10370             instr->type == INSTR_ALU_SUB_MI ||
10371             instr->type == INSTR_ALU_SUB_HI ||
10372             instr->type == INSTR_ALU_SHL_MI ||
10373             instr->type == INSTR_ALU_SHL_HI ||
10374             instr->type == INSTR_ALU_SHR_MI ||
10375             instr->type == INSTR_ALU_SHR_HI ||
10376             instr->type == INSTR_ALU_AND_I ||
10377             instr->type == INSTR_ALU_OR_I ||
10378             instr->type == INSTR_ALU_XOR_I)
10379                 imm = 1;
10380
10381         if (!imm)
10382                 fprintf(f,
10383                         "\t{\n"
10384                         "\t\t.type = %s,\n"
10385                         "\t\t.alu = {\n"
10386                         "\t\t\t.dst = {\n"
10387                         "\t\t\t\t.struct_id = %u,\n"
10388                         "\t\t\t\t.n_bits = %u,\n"
10389                         "\t\t\t\t.offset = %u,\n"
10390                         "\t\t\t},\n"
10391                         "\t\t\t.src = {\n"
10392                         "\t\t\t\t.struct_id = %u,\n"
10393                         "\t\t\t\t.n_bits = %u,\n"
10394                         "\t\t\t\t.offset = %u,\n"
10395                         "\t\t\t},\n"
10396                         "\t\t},\n"
10397                         "\t},\n",
10398                         instr_type_to_name(instr),
10399                         instr->alu.dst.struct_id,
10400                         instr->alu.dst.n_bits,
10401                         instr->alu.dst.offset,
10402                         instr->alu.src.struct_id,
10403                         instr->alu.src.n_bits,
10404                         instr->alu.src.offset);
10405         else
10406                 fprintf(f,
10407                         "\t{\n"
10408                         "\t\t.type = %s,\n"
10409                         "\t\t.alu = {\n"
10410                         "\t\t\t.dst = {\n"
10411                         "\t\t\t\t.struct_id = %u,\n"
10412                         "\t\t\t\t.n_bits = %u,\n"
10413                         "\t\t\t\t.offset = %u,\n"
10414                         "\t\t\t}\n,"
10415                         "\t\t\t.src_val = %" PRIu64 ",\n"
10416                         "\t\t},\n"
10417                         "\t},\n",
10418                         instr_type_to_name(instr),
10419                         instr->alu.dst.struct_id,
10420                         instr->alu.dst.n_bits,
10421                         instr->alu.dst.offset,
10422                         instr->alu.src_val);
10423 }
10424
10425 static void
10426 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10427 {
10428         int prefetch  = 0, idx_imm = 0, src_imm = 0;
10429
10430         if (instr->type == INSTR_REGPREFETCH_RH ||
10431             instr->type == INSTR_REGPREFETCH_RM ||
10432             instr->type == INSTR_REGPREFETCH_RI)
10433                 prefetch = 1;
10434
10435         /* index is the 3rd operand for the regrd instruction and the 2nd
10436          * operand for the regwr and regadd instructions.
10437          */
10438         if (instr->type == INSTR_REGPREFETCH_RI ||
10439             instr->type == INSTR_REGRD_HRI ||
10440             instr->type == INSTR_REGRD_MRI ||
10441             instr->type == INSTR_REGWR_RIH ||
10442             instr->type == INSTR_REGWR_RIM ||
10443             instr->type == INSTR_REGWR_RII ||
10444             instr->type == INSTR_REGADD_RIH ||
10445             instr->type == INSTR_REGADD_RIM ||
10446             instr->type == INSTR_REGADD_RII)
10447                 idx_imm = 1;
10448
10449         /* src is the 3rd operand for the regwr and regadd instructions. */
10450         if (instr->type == INSTR_REGWR_RHI ||
10451             instr->type == INSTR_REGWR_RMI ||
10452             instr->type == INSTR_REGWR_RII ||
10453             instr->type == INSTR_REGADD_RHI ||
10454             instr->type == INSTR_REGADD_RMI ||
10455             instr->type == INSTR_REGADD_RII)
10456                 src_imm = 1;
10457
10458         /* instr.regarray.regarray_id. */
10459         fprintf(f,
10460                 "\t{\n"
10461                 "\t\t.type = %s,\n"
10462                 "\t\t.regarray = {\n"
10463                 "\t\t\t.regarray_id = %u,\n",
10464                 instr_type_to_name(instr),
10465                 instr->regarray.regarray_id);
10466
10467         /* instr.regarray.idx / instr.regarray.idx_val. */
10468         if (!idx_imm)
10469                 fprintf(f,
10470                         "\t\t\t\t.idx = {\n"
10471                         "\t\t\t\t\t.struct_id = %u,\n"
10472                         "\t\t\t\t\t.n_bits = %u,\n"
10473                         "\t\t\t\t\t.offset = %u,\n"
10474                         "\t\t\t\t},\n",
10475                         instr->regarray.idx.struct_id,
10476                         instr->regarray.idx.n_bits,
10477                         instr->regarray.idx.offset);
10478         else
10479                 fprintf(f,
10480                         "\t\t\t\t.idx_val = %u,\n",
10481                         instr->regarray.idx_val);
10482
10483         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10484         if (!prefetch) {
10485                 if (!src_imm)
10486                         fprintf(f,
10487                                 "\t\t\t\t.dstsrc = {\n"
10488                                 "\t\t\t\t\t.struct_id = %u,\n"
10489                                 "\t\t\t\t\t.n_bits = %u,\n"
10490                                 "\t\t\t\t\t.offset = %u,\n"
10491                                 "\t\t\t\t},\n",
10492                                 instr->regarray.dstsrc.struct_id,
10493                                 instr->regarray.dstsrc.n_bits,
10494                                 instr->regarray.dstsrc.offset);
10495                 else
10496                         fprintf(f,
10497                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10498                                 instr->regarray.dstsrc_val);
10499         }
10500
10501         /* instr.regarray and instr - closing curly braces. */
10502         fprintf(f,
10503                 "\t\t},\n"
10504                 "\t},\n");
10505 }
10506
10507 static void
10508 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10509 {
10510         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
10511
10512         if (instr->type == INSTR_METPREFETCH_H ||
10513             instr->type == INSTR_METPREFETCH_M ||
10514             instr->type == INSTR_METPREFETCH_I)
10515                 prefetch = 1;
10516
10517         /* idx_imm. */
10518         if (instr->type == INSTR_METPREFETCH_I ||
10519             instr->type == INSTR_METER_IHM ||
10520             instr->type == INSTR_METER_IHI ||
10521             instr->type == INSTR_METER_IMM ||
10522             instr->type == INSTR_METER_IMI)
10523                 idx_imm = 1;
10524
10525         /* color_in_imm. */
10526         if (instr->type == INSTR_METER_HHI ||
10527             instr->type == INSTR_METER_HMI ||
10528             instr->type == INSTR_METER_MHI ||
10529             instr->type == INSTR_METER_MMI ||
10530             instr->type == INSTR_METER_IHI ||
10531             instr->type == INSTR_METER_IMI)
10532                 color_in_imm = 1;
10533
10534         /* instr.meter.metarray_id. */
10535         fprintf(f,
10536                 "\t{\n"
10537                 "\t\t.type = %s,\n"
10538                 "\t\t.meter = {\n"
10539                 "\t\t\t.metarray_id = %u,\n",
10540                 instr_type_to_name(instr),
10541                 instr->meter.metarray_id);
10542
10543         /* instr.meter.idx / instr.meter.idx_val. */
10544         if (!idx_imm)
10545                 fprintf(f,
10546                         "\t\t\t.idx = {\n"
10547                         "\t\t\t\t.struct_id = %u,\n"
10548                         "\t\t\t\t.n_bits = %u,\n"
10549                         "\t\t\t\t.offset = %u,\n"
10550                         "\t\t\t},\n",
10551                         instr->meter.idx.struct_id,
10552                         instr->meter.idx.n_bits,
10553                         instr->meter.idx.offset);
10554         else
10555                 fprintf(f,
10556                         "\t\t\t.idx_val = %u,\n",
10557                         instr->meter.idx_val);
10558
10559         if (!prefetch) {
10560                 /* instr.meter.length. */
10561                 fprintf(f,
10562                         "\t\t\t.length = {\n"
10563                         "\t\t\t\t.struct_id = %u,\n"
10564                         "\t\t\t\t.n_bits = %u,\n"
10565                         "\t\t\t\t.offset = %u,\n"
10566                         "\t\t\t},\n",
10567                         instr->meter.length.struct_id,
10568                         instr->meter.length.n_bits,
10569                         instr->meter.length.offset);
10570
10571                 /* instr.meter.color_in / instr.meter.color_in_val. */
10572                 if (!color_in_imm)
10573                         fprintf(f,
10574                                 "\t\t\t.color_in = {\n"
10575                                 "\t\t\t\t.struct_id = %u,\n"
10576                                 "\t\t\t\t.n_bits = %u,\n"
10577                                 "\t\t\t\t.offset = %u,\n"
10578                                 "\t\t\t},\n",
10579                                 instr->meter.color_in.struct_id,
10580                                 instr->meter.color_in.n_bits,
10581                                 instr->meter.color_in.offset);
10582                 else
10583                         fprintf(f,
10584                                 "\t\t\t.color_in_val = %u,\n",
10585                                 (uint32_t)instr->meter.color_in_val);
10586
10587                 /* instr.meter.color_out. */
10588                 fprintf(f,
10589                         "\t\t\t.color_out = {\n"
10590                         "\t\t\t\t.struct_id = %u,\n"
10591                         "\t\t\t\t.n_bits = %u,\n"
10592                         "\t\t\t\t.offset = %u,\n"
10593                         "\t\t\t},\n",
10594                         instr->meter.color_out.struct_id,
10595                         instr->meter.color_out.n_bits,
10596                         instr->meter.color_out.offset);
10597         }
10598
10599         /* instr.meter and instr - closing curly braces. */
10600         fprintf(f,
10601                 "\t\t},\n"
10602                 "\t},\n");
10603 }
10604
10605 static void
10606 instr_table_export(struct instruction *instr,
10607                 FILE *f)
10608 {
10609         fprintf(f,
10610                 "\t{\n"
10611                 "\t\t.type = %s,\n"
10612                 "\t\t.table = {\n"
10613                 "\t\t\t.table_id = %u,\n"
10614                 "\t\t},\n"
10615                 "\t},\n",
10616                 instr_type_to_name(instr),
10617                 instr->table.table_id);
10618 }
10619
10620 static void
10621 instr_learn_export(struct instruction *instr, FILE *f)
10622 {
10623         fprintf(f,
10624                 "\t{\n"
10625                 "\t\t.type = %s,\n"
10626                 "\t\t.learn = {\n"
10627                 "\t\t\t\t.action_id = %u,\n"
10628                 "\t\t},\n"
10629                 "\t},\n",
10630                 instr_type_to_name(instr),
10631                 instr->learn.action_id);
10632 }
10633
10634 static void
10635 instr_forget_export(struct instruction *instr, FILE *f)
10636 {
10637         fprintf(f,
10638                 "\t{\n"
10639                 "\t\t.type = %s,\n"
10640                 "\t},\n",
10641                 instr_type_to_name(instr));
10642 }
10643
10644 static void
10645 instr_extern_export(struct instruction *instr, FILE *f)
10646 {
10647         if (instr->type == INSTR_EXTERN_OBJ)
10648                 fprintf(f,
10649                         "\t{\n"
10650                         "\t\t.type = %s,\n"
10651                         "\t\t.ext_obj = {\n"
10652                         "\t\t\t.ext_obj_id = %u,\n"
10653                         "\t\t\t.func_id = %u,\n"
10654                         "\t\t},\n"
10655                         "\t},\n",
10656                         instr_type_to_name(instr),
10657                         instr->ext_obj.ext_obj_id,
10658                         instr->ext_obj.func_id);
10659         else
10660                 fprintf(f,
10661                         "\t{\n"
10662                         "\t\t.type = %s,\n"
10663                         "\t\t.ext_func = {\n"
10664                         "\t\t\t.ext_func_id = %u,\n"
10665                         "\t\t},\n"
10666                         "\t},\n",
10667                         instr_type_to_name(instr),
10668                         instr->ext_func.ext_func_id);
10669 }
10670
10671 static void
10672 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10673 {
10674         fprintf(f,
10675                 "\t{\n"
10676                 "\t\t.type = %s,\n"
10677                 "\t\t.jmp = {\n"
10678                 "\t\t\t.ip = NULL,\n",
10679                 instr_type_to_name(instr));
10680
10681         switch (instr->type) {
10682         case INSTR_JMP_VALID:
10683         case INSTR_JMP_INVALID:
10684                 fprintf(f,
10685                         "\t\t\t.header_id = %u,\n",
10686                         instr->jmp.header_id);
10687                 break;
10688
10689         case INSTR_JMP_ACTION_HIT:
10690         case INSTR_JMP_ACTION_MISS:
10691                 fprintf(f,
10692                         "\t\t\t.action_id = %u,\n",
10693                         instr->jmp.action_id);
10694                 break;
10695
10696         case INSTR_JMP_EQ:
10697         case INSTR_JMP_EQ_MH:
10698         case INSTR_JMP_EQ_HM:
10699         case INSTR_JMP_EQ_HH:
10700         case INSTR_JMP_NEQ:
10701         case INSTR_JMP_NEQ_MH:
10702         case INSTR_JMP_NEQ_HM:
10703         case INSTR_JMP_NEQ_HH:
10704         case INSTR_JMP_LT:
10705         case INSTR_JMP_LT_MH:
10706         case INSTR_JMP_LT_HM:
10707         case INSTR_JMP_LT_HH:
10708         case INSTR_JMP_GT:
10709         case INSTR_JMP_GT_MH:
10710         case INSTR_JMP_GT_HM:
10711         case INSTR_JMP_GT_HH:
10712                 fprintf(f,
10713                         "\t\t\t.a = {\n"
10714                         "\t\t\t\t.struct_id = %u,\n"
10715                         "\t\t\t\t.n_bits = %u,\n"
10716                         "\t\t\t\t.offset = %u,\n"
10717                         "\t\t\t},\n"
10718                         "\t\t\t.b = {\n"
10719                         "\t\t\t\t.struct_id = %u,\n"
10720                         "\t\t\t\t.n_bits = %u,\n"
10721                         "\t\t\t\t.offset = %u,\n"
10722                         "\t\t\t},\n",
10723                         instr->jmp.a.struct_id,
10724                         instr->jmp.a.n_bits,
10725                         instr->jmp.a.offset,
10726                         instr->jmp.b.struct_id,
10727                         instr->jmp.b.n_bits,
10728                         instr->jmp.b.offset);
10729                 break;
10730
10731         case INSTR_JMP_EQ_I:
10732         case INSTR_JMP_NEQ_I:
10733         case INSTR_JMP_LT_MI:
10734         case INSTR_JMP_LT_HI:
10735         case INSTR_JMP_GT_MI:
10736         case INSTR_JMP_GT_HI:
10737                 fprintf(f,
10738                         "\t\t\t.a = {\n"
10739                         "\t\t\t\t.struct_id = %u,\n"
10740                         "\t\t\t\t.n_bits = %u,\n"
10741                         "\t\t\t\t.offset = %u,\n"
10742                         "\t\t\t}\n,"
10743                         "\t\t\t.b_val = %" PRIu64 ",\n",
10744                         instr->jmp.a.struct_id,
10745                         instr->jmp.a.n_bits,
10746                         instr->jmp.a.offset,
10747                         instr->jmp.b_val);
10748                 break;
10749
10750         default:
10751                 break;
10752         }
10753
10754         fprintf(f,
10755                 "\t\t},\n"
10756                 "\t},\n");
10757 }
10758
10759 static void
10760 instr_return_export(struct instruction *instr,
10761                 FILE *f)
10762 {
10763         fprintf(f,
10764                 "\t{\n"
10765                 "\t\t.type = %s,\n",
10766                 instr_type_to_name(instr));
10767
10768         fprintf(f,
10769                 "\t},\n");
10770 }
10771
10772 static instruction_export_t export_table[] = {
10773         [INSTR_RX] = instr_io_export,
10774
10775         [INSTR_TX] = instr_io_export,
10776         [INSTR_TX_I] = instr_io_export,
10777
10778         [INSTR_HDR_EXTRACT] = instr_io_export,
10779         [INSTR_HDR_EXTRACT2] = instr_io_export,
10780         [INSTR_HDR_EXTRACT3] = instr_io_export,
10781         [INSTR_HDR_EXTRACT4] = instr_io_export,
10782         [INSTR_HDR_EXTRACT5] = instr_io_export,
10783         [INSTR_HDR_EXTRACT6] = instr_io_export,
10784         [INSTR_HDR_EXTRACT7] = instr_io_export,
10785         [INSTR_HDR_EXTRACT8] = instr_io_export,
10786
10787         [INSTR_HDR_EXTRACT_M] = instr_io_export,
10788
10789         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10790
10791         [INSTR_HDR_EMIT] = instr_io_export,
10792         [INSTR_HDR_EMIT_TX] = instr_io_export,
10793         [INSTR_HDR_EMIT2_TX] = instr_io_export,
10794         [INSTR_HDR_EMIT3_TX] = instr_io_export,
10795         [INSTR_HDR_EMIT4_TX] = instr_io_export,
10796         [INSTR_HDR_EMIT5_TX] = instr_io_export,
10797         [INSTR_HDR_EMIT6_TX] = instr_io_export,
10798         [INSTR_HDR_EMIT7_TX] = instr_io_export,
10799         [INSTR_HDR_EMIT8_TX] = instr_io_export,
10800
10801         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10802         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10803
10804         [INSTR_MOV] = instr_mov_export,
10805         [INSTR_MOV_MH] = instr_mov_export,
10806         [INSTR_MOV_HM] = instr_mov_export,
10807         [INSTR_MOV_HH] = instr_mov_export,
10808         [INSTR_MOV_I] = instr_mov_export,
10809
10810         [INSTR_DMA_HT]  = instr_dma_ht_export,
10811         [INSTR_DMA_HT2] = instr_dma_ht_export,
10812         [INSTR_DMA_HT3] = instr_dma_ht_export,
10813         [INSTR_DMA_HT4] = instr_dma_ht_export,
10814         [INSTR_DMA_HT5] = instr_dma_ht_export,
10815         [INSTR_DMA_HT6] = instr_dma_ht_export,
10816         [INSTR_DMA_HT7] = instr_dma_ht_export,
10817         [INSTR_DMA_HT8] = instr_dma_ht_export,
10818
10819         [INSTR_ALU_ADD] = instr_alu_export,
10820         [INSTR_ALU_ADD_MH] = instr_alu_export,
10821         [INSTR_ALU_ADD_HM] = instr_alu_export,
10822         [INSTR_ALU_ADD_HH] = instr_alu_export,
10823         [INSTR_ALU_ADD_MI] = instr_alu_export,
10824         [INSTR_ALU_ADD_HI] = instr_alu_export,
10825
10826         [INSTR_ALU_SUB] = instr_alu_export,
10827         [INSTR_ALU_SUB_MH] = instr_alu_export,
10828         [INSTR_ALU_SUB_HM] = instr_alu_export,
10829         [INSTR_ALU_SUB_HH] = instr_alu_export,
10830         [INSTR_ALU_SUB_MI] = instr_alu_export,
10831         [INSTR_ALU_SUB_HI] = instr_alu_export,
10832
10833         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10834         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10835         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10836         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10837
10838         [INSTR_ALU_AND] = instr_alu_export,
10839         [INSTR_ALU_AND_MH] = instr_alu_export,
10840         [INSTR_ALU_AND_HM] = instr_alu_export,
10841         [INSTR_ALU_AND_HH] = instr_alu_export,
10842         [INSTR_ALU_AND_I] = instr_alu_export,
10843
10844         [INSTR_ALU_OR] = instr_alu_export,
10845         [INSTR_ALU_OR_MH] = instr_alu_export,
10846         [INSTR_ALU_OR_HM] = instr_alu_export,
10847         [INSTR_ALU_OR_HH] = instr_alu_export,
10848         [INSTR_ALU_OR_I] = instr_alu_export,
10849
10850         [INSTR_ALU_XOR] = instr_alu_export,
10851         [INSTR_ALU_XOR_MH] = instr_alu_export,
10852         [INSTR_ALU_XOR_HM] = instr_alu_export,
10853         [INSTR_ALU_XOR_HH] = instr_alu_export,
10854         [INSTR_ALU_XOR_I] = instr_alu_export,
10855
10856         [INSTR_ALU_SHL] = instr_alu_export,
10857         [INSTR_ALU_SHL_MH] = instr_alu_export,
10858         [INSTR_ALU_SHL_HM] = instr_alu_export,
10859         [INSTR_ALU_SHL_HH] = instr_alu_export,
10860         [INSTR_ALU_SHL_MI] = instr_alu_export,
10861         [INSTR_ALU_SHL_HI] = instr_alu_export,
10862
10863         [INSTR_ALU_SHR] = instr_alu_export,
10864         [INSTR_ALU_SHR_MH] = instr_alu_export,
10865         [INSTR_ALU_SHR_HM] = instr_alu_export,
10866         [INSTR_ALU_SHR_HH] = instr_alu_export,
10867         [INSTR_ALU_SHR_MI] = instr_alu_export,
10868         [INSTR_ALU_SHR_HI] = instr_alu_export,
10869
10870         [INSTR_REGPREFETCH_RH] = instr_reg_export,
10871         [INSTR_REGPREFETCH_RM] = instr_reg_export,
10872         [INSTR_REGPREFETCH_RI] = instr_reg_export,
10873
10874         [INSTR_REGRD_HRH] = instr_reg_export,
10875         [INSTR_REGRD_HRM] = instr_reg_export,
10876         [INSTR_REGRD_MRH] = instr_reg_export,
10877         [INSTR_REGRD_MRM] = instr_reg_export,
10878         [INSTR_REGRD_HRI] = instr_reg_export,
10879         [INSTR_REGRD_MRI] = instr_reg_export,
10880
10881         [INSTR_REGWR_RHH] = instr_reg_export,
10882         [INSTR_REGWR_RHM] = instr_reg_export,
10883         [INSTR_REGWR_RMH] = instr_reg_export,
10884         [INSTR_REGWR_RMM] = instr_reg_export,
10885         [INSTR_REGWR_RHI] = instr_reg_export,
10886         [INSTR_REGWR_RMI] = instr_reg_export,
10887         [INSTR_REGWR_RIH] = instr_reg_export,
10888         [INSTR_REGWR_RIM] = instr_reg_export,
10889         [INSTR_REGWR_RII] = instr_reg_export,
10890
10891         [INSTR_REGADD_RHH] = instr_reg_export,
10892         [INSTR_REGADD_RHM] = instr_reg_export,
10893         [INSTR_REGADD_RMH] = instr_reg_export,
10894         [INSTR_REGADD_RMM] = instr_reg_export,
10895         [INSTR_REGADD_RHI] = instr_reg_export,
10896         [INSTR_REGADD_RMI] = instr_reg_export,
10897         [INSTR_REGADD_RIH] = instr_reg_export,
10898         [INSTR_REGADD_RIM] = instr_reg_export,
10899         [INSTR_REGADD_RII] = instr_reg_export,
10900
10901         [INSTR_METPREFETCH_H] = instr_meter_export,
10902         [INSTR_METPREFETCH_M] = instr_meter_export,
10903         [INSTR_METPREFETCH_I] = instr_meter_export,
10904
10905         [INSTR_METER_HHM] = instr_meter_export,
10906         [INSTR_METER_HHI] = instr_meter_export,
10907         [INSTR_METER_HMM] = instr_meter_export,
10908         [INSTR_METER_HMI] = instr_meter_export,
10909         [INSTR_METER_MHM] = instr_meter_export,
10910         [INSTR_METER_MHI] = instr_meter_export,
10911         [INSTR_METER_MMM] = instr_meter_export,
10912         [INSTR_METER_MMI] = instr_meter_export,
10913         [INSTR_METER_IHM] = instr_meter_export,
10914         [INSTR_METER_IHI] = instr_meter_export,
10915         [INSTR_METER_IMM] = instr_meter_export,
10916         [INSTR_METER_IMI] = instr_meter_export,
10917
10918         [INSTR_TABLE] = instr_table_export,
10919         [INSTR_TABLE_AF] = instr_table_export,
10920         [INSTR_SELECTOR] = instr_table_export,
10921         [INSTR_LEARNER] = instr_table_export,
10922         [INSTR_LEARNER_AF] = instr_table_export,
10923
10924         [INSTR_LEARNER_LEARN] = instr_learn_export,
10925         [INSTR_LEARNER_FORGET] = instr_forget_export,
10926
10927         [INSTR_EXTERN_OBJ] = instr_extern_export,
10928         [INSTR_EXTERN_FUNC] = instr_extern_export,
10929
10930         [INSTR_JMP] = instr_jmp_export,
10931         [INSTR_JMP_VALID] = instr_jmp_export,
10932         [INSTR_JMP_INVALID] = instr_jmp_export,
10933         [INSTR_JMP_HIT] = instr_jmp_export,
10934         [INSTR_JMP_MISS] = instr_jmp_export,
10935         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
10936         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
10937
10938         [INSTR_JMP_EQ] = instr_jmp_export,
10939         [INSTR_JMP_EQ_MH] = instr_jmp_export,
10940         [INSTR_JMP_EQ_HM] = instr_jmp_export,
10941         [INSTR_JMP_EQ_HH] = instr_jmp_export,
10942         [INSTR_JMP_EQ_I] = instr_jmp_export,
10943
10944         [INSTR_JMP_NEQ] = instr_jmp_export,
10945         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
10946         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
10947         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
10948         [INSTR_JMP_NEQ_I] = instr_jmp_export,
10949
10950         [INSTR_JMP_LT] = instr_jmp_export,
10951         [INSTR_JMP_LT_MH] = instr_jmp_export,
10952         [INSTR_JMP_LT_HM] = instr_jmp_export,
10953         [INSTR_JMP_LT_HH] = instr_jmp_export,
10954         [INSTR_JMP_LT_MI] = instr_jmp_export,
10955         [INSTR_JMP_LT_HI] = instr_jmp_export,
10956
10957         [INSTR_JMP_GT] = instr_jmp_export,
10958         [INSTR_JMP_GT_MH] = instr_jmp_export,
10959         [INSTR_JMP_GT_HM] = instr_jmp_export,
10960         [INSTR_JMP_GT_HH] = instr_jmp_export,
10961         [INSTR_JMP_GT_MI] = instr_jmp_export,
10962         [INSTR_JMP_GT_HI] = instr_jmp_export,
10963
10964         [INSTR_RETURN] = instr_return_export,
10965 };
10966
10967 static void
10968 action_data_codegen(struct action *a, FILE *f)
10969 {
10970         uint32_t i;
10971
10972         fprintf(f,
10973                 "static const struct instruction action_%s_instructions[] = {\n",
10974                 a->name);
10975
10976         for (i = 0; i < a->n_instructions; i++) {
10977                 struct instruction *instr = &a->instructions[i];
10978                 instruction_export_t func = export_table[instr->type];
10979
10980                 func(instr, f);
10981         }
10982
10983         fprintf(f, "};\n");
10984 }
10985
10986 static const char *
10987 instr_type_to_func(struct instruction *instr)
10988 {
10989         switch (instr->type) {
10990         case INSTR_RX: return NULL;
10991
10992         case INSTR_TX: return "__instr_tx_exec";
10993         case INSTR_TX_I: return "__instr_tx_i_exec";
10994
10995         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
10996         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
10997         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
10998         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
10999         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11000         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11001         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11002         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11003
11004         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11005
11006         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11007
11008         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11009         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11010         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11011         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11012         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11013         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11014         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11015         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11016         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11017
11018         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11019         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11020
11021         case INSTR_MOV: return "__instr_mov_exec";
11022         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11023         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11024         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11025         case INSTR_MOV_I: return "__instr_mov_i_exec";
11026
11027         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11028         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11029         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11030         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11031         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11032         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11033         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11034         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11035
11036         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11037         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11038         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11039         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11040         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11041         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11042
11043         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11044         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11045         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11046         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11047         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11048         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11049
11050         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11051         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11052         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11053         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11054
11055         case INSTR_ALU_AND: return "__instr_alu_and_exec";
11056         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11057         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11058         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11059         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11060
11061         case INSTR_ALU_OR: return "__instr_alu_or_exec";
11062         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11063         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11064         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11065         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11066
11067         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11068         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11069         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11070         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11071         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11072
11073         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11074         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11075         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11076         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11077         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11078         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11079
11080         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11081         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11082         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11083         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11084         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11085         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11086
11087         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11088         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11089         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11090
11091         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11092         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11093         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11094         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11095         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11096         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11097
11098         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11099         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11100         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11101         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11102         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11103         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11104         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11105         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11106         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11107
11108         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11109         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11110         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11111         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11112         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11113         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11114         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11115         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11116         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11117
11118         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11119         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11120         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11121
11122         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11123         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11124         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11125         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11126         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11127         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11128         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11129         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11130         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11131         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11132         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11133         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11134
11135         case INSTR_TABLE: return NULL;
11136         case INSTR_TABLE_AF: return NULL;
11137         case INSTR_SELECTOR: return NULL;
11138         case INSTR_LEARNER: return NULL;
11139         case INSTR_LEARNER_AF: return NULL;
11140
11141         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11142         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11143
11144         case INSTR_EXTERN_OBJ: return NULL;
11145         case INSTR_EXTERN_FUNC: return NULL;
11146
11147         case INSTR_JMP: return NULL;
11148         case INSTR_JMP_VALID: return NULL;
11149         case INSTR_JMP_INVALID: return NULL;
11150         case INSTR_JMP_HIT: return NULL;
11151         case INSTR_JMP_MISS: return NULL;
11152         case INSTR_JMP_ACTION_HIT: return NULL;
11153         case INSTR_JMP_ACTION_MISS: return NULL;
11154         case INSTR_JMP_EQ: return NULL;
11155         case INSTR_JMP_EQ_MH: return NULL;
11156         case INSTR_JMP_EQ_HM: return NULL;
11157         case INSTR_JMP_EQ_HH: return NULL;
11158         case INSTR_JMP_EQ_I: return NULL;
11159         case INSTR_JMP_NEQ: return NULL;
11160         case INSTR_JMP_NEQ_MH: return NULL;
11161         case INSTR_JMP_NEQ_HM: return NULL;
11162         case INSTR_JMP_NEQ_HH: return NULL;
11163         case INSTR_JMP_NEQ_I: return NULL;
11164         case INSTR_JMP_LT: return NULL;
11165         case INSTR_JMP_LT_MH: return NULL;
11166         case INSTR_JMP_LT_HM: return NULL;
11167         case INSTR_JMP_LT_HH: return NULL;
11168         case INSTR_JMP_LT_MI: return NULL;
11169         case INSTR_JMP_LT_HI: return NULL;
11170         case INSTR_JMP_GT: return NULL;
11171         case INSTR_JMP_GT_MH: return NULL;
11172         case INSTR_JMP_GT_HM: return NULL;
11173         case INSTR_JMP_GT_HH: return NULL;
11174         case INSTR_JMP_GT_MI: return NULL;
11175         case INSTR_JMP_GT_HI: return NULL;
11176
11177         case INSTR_RETURN: return NULL;
11178
11179         default: return NULL;
11180         }
11181 }
11182
11183 static void
11184 action_instr_does_tx_codegen(struct action *a,
11185                         uint32_t instr_pos,
11186                         struct instruction *instr,
11187                         FILE *f)
11188 {
11189         fprintf(f,
11190                 "%s(p, t, &action_%s_instructions[%u]);\n"
11191                 "\tthread_ip_reset(p, t);\n"
11192                 "\tinstr_rx_exec(p);\n"
11193                 "\treturn;\n",
11194                 instr_type_to_func(instr),
11195                 a->name,
11196                 instr_pos);
11197 }
11198
11199 static void
11200 action_instr_extern_obj_codegen(struct action *a,
11201                                 uint32_t instr_pos,
11202                                 FILE *f)
11203 {
11204         fprintf(f,
11205                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11206                 a->name,
11207                 instr_pos);
11208 }
11209
11210 static void
11211 action_instr_extern_func_codegen(struct action *a,
11212                                  uint32_t instr_pos,
11213                                  FILE *f)
11214 {
11215         fprintf(f,
11216                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11217                 a->name,
11218                 instr_pos);
11219 }
11220
11221 static void
11222 action_instr_jmp_codegen(struct action *a,
11223                          uint32_t instr_pos,
11224                          struct instruction *instr,
11225                          struct instruction_data *data,
11226                          FILE *f)
11227 {
11228         switch (instr->type) {
11229         case INSTR_JMP:
11230                 fprintf(f,
11231                         "goto %s;\n",
11232                         data->jmp_label);
11233                 return;
11234
11235         case INSTR_JMP_VALID:
11236                 fprintf(f,
11237                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11238                         "\t\tgoto %s;\n",
11239                         a->name,
11240                         instr_pos,
11241                         data->jmp_label);
11242                 return;
11243
11244         case INSTR_JMP_INVALID:
11245                 fprintf(f,
11246                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11247                         "\t\tgoto %s;\n",
11248                         a->name,
11249                         instr_pos,
11250                         data->jmp_label);
11251                 return;
11252
11253         case INSTR_JMP_HIT:
11254                 fprintf(f,
11255                         "if (t->hit)\n"
11256                         "\t\tgoto %s;\n",
11257                         data->jmp_label);
11258                 return;
11259
11260         case INSTR_JMP_MISS:
11261                 fprintf(f,
11262                         "if (!t->hit)\n"
11263                         "\t\tgoto %s;\n",
11264                         data->jmp_label);
11265                 return;
11266
11267         case INSTR_JMP_ACTION_HIT:
11268                 fprintf(f,
11269                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11270                         "\t\tgoto %s;\n",
11271                         a->name,
11272                         instr_pos,
11273                         data->jmp_label);
11274                 return;
11275
11276         case INSTR_JMP_ACTION_MISS:
11277                 fprintf(f,
11278                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11279                         "\t\tgoto %s;\n",
11280                         a->name,
11281                         instr_pos,
11282                         data->jmp_label);
11283                 return;
11284
11285         case INSTR_JMP_EQ:
11286                 fprintf(f,
11287                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11288                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11289                         "\t\tgoto %s;\n",
11290                         a->name,
11291                         instr_pos,
11292                         a->name,
11293                         instr_pos,
11294                         data->jmp_label);
11295                 return;
11296
11297         case INSTR_JMP_EQ_MH:
11298                 fprintf(f,
11299                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11300                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11301                         "\t\tgoto %s;\n",
11302                         a->name,
11303                         instr_pos,
11304                         a->name,
11305                         instr_pos,
11306                         data->jmp_label);
11307                 return;
11308
11309         case INSTR_JMP_EQ_HM:
11310                 fprintf(f,
11311                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11312                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11313                         "\t\tgoto %s;\n",
11314                         a->name,
11315                         instr_pos,
11316                         a->name,
11317                         instr_pos,
11318                         data->jmp_label);
11319                 return;
11320
11321         case INSTR_JMP_EQ_HH:
11322                 fprintf(f,
11323                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11324                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11325                         "\t\tgoto %s;\n",
11326                         a->name,
11327                         instr_pos,
11328                         a->name,
11329                         instr_pos,
11330                         data->jmp_label);
11331                 return;
11332
11333         case INSTR_JMP_EQ_I:
11334                 fprintf(f,
11335                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11336                         "action_%s_instructions[%u].jmp.b_val)\n"
11337                         "\t\tgoto %s;\n",
11338                         a->name,
11339                         instr_pos,
11340                         a->name,
11341                         instr_pos,
11342                         data->jmp_label);
11343                 return;
11344
11345         case INSTR_JMP_NEQ:
11346                 fprintf(f,
11347                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11348                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11349                         "\t\tgoto %s;\n",
11350                         a->name,
11351                         instr_pos,
11352                         a->name,
11353                         instr_pos,
11354                         data->jmp_label);
11355                 return;
11356
11357         case INSTR_JMP_NEQ_MH:
11358                 fprintf(f,
11359                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11360                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11361                         "\t\tgoto %s;\n",
11362                         a->name,
11363                         instr_pos,
11364                         a->name,
11365                         instr_pos,
11366                         data->jmp_label);
11367                 return;
11368
11369         case INSTR_JMP_NEQ_HM:
11370                 fprintf(f,
11371                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11372                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11373                         "\t\tgoto %s;\n",
11374                         a->name,
11375                         instr_pos,
11376                         a->name,
11377                         instr_pos,
11378                         data->jmp_label);
11379                 return;
11380
11381         case INSTR_JMP_NEQ_HH:
11382                 fprintf(f,
11383                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11384                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11385                         "\t\tgoto %s;\n",
11386                         a->name,
11387                         instr_pos,
11388                         a->name,
11389                         instr_pos,
11390                         data->jmp_label);
11391                 return;
11392
11393         case INSTR_JMP_NEQ_I:
11394                 fprintf(f,
11395                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11396                         "action_%s_instructions[%u].jmp.b_val)\n"
11397                         "\t\tgoto %s;\n",
11398                         a->name,
11399                         instr_pos,
11400                         a->name,
11401                         instr_pos,
11402                         data->jmp_label);
11403                 return;
11404
11405         case INSTR_JMP_LT:
11406                 fprintf(f,
11407                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11408                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11409                         "\t\tgoto %s;\n",
11410                         a->name,
11411                         instr_pos,
11412                         a->name,
11413                         instr_pos,
11414                         data->jmp_label);
11415                 return;
11416
11417         case INSTR_JMP_LT_MH:
11418                 fprintf(f,
11419                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11420                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11421                         "\t\tgoto %s;\n",
11422                         a->name,
11423                         instr_pos,
11424                         a->name,
11425                         instr_pos,
11426                         data->jmp_label);
11427                 return;
11428
11429         case INSTR_JMP_LT_HM:
11430                 fprintf(f,
11431                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11432                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11433                         "\t\tgoto %s;\n",
11434                         a->name,
11435                         instr_pos,
11436                         a->name,
11437                         instr_pos,
11438                         data->jmp_label);
11439                 return;
11440
11441         case INSTR_JMP_LT_HH:
11442                 fprintf(f,
11443                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11444                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11445                         "\t\tgoto %s;\n",
11446                         a->name,
11447                         instr_pos,
11448                         a->name,
11449                         instr_pos,
11450                         data->jmp_label);
11451                 return;
11452
11453         case INSTR_JMP_LT_MI:
11454                 fprintf(f,
11455                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11456                         "action_%s_instructions[%u].jmp.b_val)\n"
11457                         "\t\tgoto %s;\n",
11458                         a->name,
11459                         instr_pos,
11460                         a->name,
11461                         instr_pos,
11462                         data->jmp_label);
11463                 return;
11464
11465         case INSTR_JMP_LT_HI:
11466                 fprintf(f,
11467                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11468                         "action_%s_instructions[%u].jmp.b_val)\n"
11469                         "\t\tgoto %s;\n",
11470                         a->name,
11471                         instr_pos,
11472                         a->name,
11473                         instr_pos,
11474                         data->jmp_label);
11475                 return;
11476
11477         case INSTR_JMP_GT:
11478                 fprintf(f,
11479                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11480                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11481                         "\t\tgoto %s;\n",
11482                         a->name,
11483                         instr_pos,
11484                         a->name,
11485                         instr_pos,
11486                         data->jmp_label);
11487                 return;
11488
11489         case INSTR_JMP_GT_MH:
11490                 fprintf(f,
11491                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11492                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11493                         "\t\tgoto %s;\n",
11494                         a->name,
11495                         instr_pos,
11496                         a->name,
11497                         instr_pos,
11498                         data->jmp_label);
11499                 return;
11500
11501         case INSTR_JMP_GT_HM:
11502                 fprintf(f,
11503                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11504                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11505                         "\t\tgoto %s;\n",
11506                         a->name,
11507                         instr_pos,
11508                         a->name,
11509                         instr_pos,
11510                         data->jmp_label);
11511                 return;
11512
11513         case INSTR_JMP_GT_HH:
11514                 fprintf(f,
11515                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11516                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11517                         "\t\tgoto %s;\n",
11518                         a->name,
11519                         instr_pos,
11520                         a->name,
11521                         instr_pos,
11522                         data->jmp_label);
11523                 return;
11524
11525         case INSTR_JMP_GT_MI:
11526                 fprintf(f,
11527                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11528                         "action_%s_instructions[%u].jmp.b_val)\n"
11529                         "\t\tgoto %s;\n",
11530                         a->name,
11531                         instr_pos,
11532                         a->name,
11533                         instr_pos,
11534                         data->jmp_label);
11535                 return;
11536
11537         case INSTR_JMP_GT_HI:
11538                 fprintf(f,
11539                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11540                         "action_%s_instructions[%u].jmp.b_val)\n"
11541                         "\t\tgoto %s;\n",
11542                         a->name,
11543                         instr_pos,
11544                         a->name,
11545                         instr_pos,
11546                         data->jmp_label);
11547                 return;
11548
11549         default:
11550                 return;
11551         }
11552 }
11553
11554 static void
11555 action_instr_return_codegen(FILE *f)
11556 {
11557         fprintf(f,
11558                 "return;\n");
11559 }
11560
11561 static void
11562 action_instr_codegen(struct action *a, FILE *f)
11563 {
11564         uint32_t i;
11565
11566         fprintf(f,
11567                 "void\n"
11568                 "action_%s_run(struct rte_swx_pipeline *p)\n"
11569                 "{\n"
11570                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11571                 "\n",
11572                 a->name);
11573
11574         for (i = 0; i < a->n_instructions; i++) {
11575                 struct instruction *instr = &a->instructions[i];
11576                 struct instruction_data *data = &a->instruction_data[i];
11577
11578                 /* Label, if present. */
11579                 if (data->label[0])
11580                         fprintf(f, "\n%s : ", data->label);
11581                 else
11582                         fprintf(f, "\n\t");
11583
11584                 /* TX instruction type. */
11585                 if (instruction_does_tx(instr)) {
11586                         action_instr_does_tx_codegen(a, i, instr, f);
11587                         continue;
11588                 }
11589
11590                 /* Extern object/function instruction type. */
11591                 if (instr->type == INSTR_EXTERN_OBJ) {
11592                         action_instr_extern_obj_codegen(a, i, f);
11593                         continue;
11594                 }
11595
11596                 if (instr->type == INSTR_EXTERN_FUNC) {
11597                         action_instr_extern_func_codegen(a, i, f);
11598                         continue;
11599                 }
11600
11601                 /* Jump instruction type. */
11602                 if (instruction_is_jmp(instr)) {
11603                         action_instr_jmp_codegen(a, i, instr, data, f);
11604                         continue;
11605                 }
11606
11607                 /* Return instruction type. */
11608                 if (instr->type == INSTR_RETURN) {
11609                         action_instr_return_codegen(f);
11610                         continue;
11611                 }
11612
11613                 /* Any other instruction type. */
11614                 fprintf(f,
11615                         "%s(p, t, &action_%s_instructions[%u]);\n",
11616                         instr_type_to_func(instr),
11617                         a->name,
11618                         i);
11619         }
11620
11621         fprintf(f, "}\n\n");
11622 }
11623
11624 struct instruction_group {
11625         TAILQ_ENTRY(instruction_group) node;
11626
11627         uint32_t group_id;
11628
11629         uint32_t first_instr_id;
11630
11631         uint32_t last_instr_id;
11632
11633         instr_exec_t func;
11634 };
11635
11636 TAILQ_HEAD(instruction_group_list, instruction_group);
11637
11638 static struct instruction_group *
11639 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11640 {
11641         struct instruction_group *g;
11642
11643         TAILQ_FOREACH(g, igl, node)
11644                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11645                         return g;
11646
11647         return NULL;
11648 }
11649
11650 static void
11651 instruction_group_list_free(struct instruction_group_list *igl)
11652 {
11653         if (!igl)
11654                 return;
11655
11656         for ( ; ; ) {
11657                 struct instruction_group *g;
11658
11659                 g = TAILQ_FIRST(igl);
11660                 if (!g)
11661                         break;
11662
11663                 TAILQ_REMOVE(igl, g, node);
11664                 free(g);
11665         }
11666
11667         free(igl);
11668 }
11669
11670 static struct instruction_group_list *
11671 instruction_group_list_create(struct rte_swx_pipeline *p)
11672 {
11673         struct instruction_group_list *igl = NULL;
11674         struct instruction_group *g = NULL;
11675         uint32_t n_groups = 0, i;
11676
11677         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11678                 goto error;
11679
11680         /* List init. */
11681         igl = calloc(1, sizeof(struct instruction_group_list));
11682         if (!igl)
11683                 goto error;
11684
11685         TAILQ_INIT(igl);
11686
11687         /* Allocate the first group. */
11688         g = calloc(1, sizeof(struct instruction_group));
11689         if (!g)
11690                 goto error;
11691
11692         /* Iteration 1: Separate the instructions into groups based on the thread yield
11693          * instructions. Do not worry about the jump instructions at this point.
11694          */
11695         for (i = 0; i < p->n_instructions; i++) {
11696                 struct instruction *instr = &p->instructions[i];
11697
11698                 /* Check for thread yield instructions. */
11699                 if (!instruction_does_thread_yield(instr))
11700                         continue;
11701
11702                 /* If the current group contains at least one instruction, then finalize it (with
11703                  * the previous instruction), add it to the list and allocate a new group (that
11704                  * starts with the current instruction).
11705                  */
11706                 if (i - g->first_instr_id) {
11707                         /* Finalize the group. */
11708                         g->last_instr_id = i - 1;
11709
11710                         /* Add the group to the list. Advance the number of groups. */
11711                         TAILQ_INSERT_TAIL(igl, g, node);
11712                         n_groups++;
11713
11714                         /* Allocate a new group. */
11715                         g = calloc(1, sizeof(struct instruction_group));
11716                         if (!g)
11717                                 goto error;
11718
11719                         /* Initialize the new group. */
11720                         g->group_id = n_groups;
11721                         g->first_instr_id = i;
11722                 }
11723
11724                 /* Finalize the current group (with the current instruction, therefore this group
11725                  * contains just the current thread yield instruction), add it to the list and
11726                  * allocate a new group (that starts with the next instruction).
11727                  */
11728
11729                 /* Finalize the group. */
11730                 g->last_instr_id = i;
11731
11732                 /* Add the group to the list. Advance the number of groups. */
11733                 TAILQ_INSERT_TAIL(igl, g, node);
11734                 n_groups++;
11735
11736                 /* Allocate a new group. */
11737                 g = calloc(1, sizeof(struct instruction_group));
11738                 if (!g)
11739                         goto error;
11740
11741                 /* Initialize the new group. */
11742                 g->group_id = n_groups;
11743                 g->first_instr_id = i + 1;
11744         }
11745
11746         /* Handle the last group. */
11747         if (i - g->first_instr_id) {
11748                 /* Finalize the group. */
11749                 g->last_instr_id = i - 1;
11750
11751                 /* Add the group to the list. Advance the number of groups. */
11752                 TAILQ_INSERT_TAIL(igl, g, node);
11753                 n_groups++;
11754         } else
11755                 free(g);
11756
11757         g = NULL;
11758
11759         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11760          * the destination of a jump instruction located in a different group ("far jump"), then the
11761          * current group has to be split, so that the instruction representing the far jump
11762          * destination is at the start of its group.
11763          */
11764         for ( ; ; ) {
11765                 int is_modified = 0;
11766
11767                 for (i = 0; i < p->n_instructions; i++) {
11768                         struct instruction_data *data = &p->instruction_data[i];
11769                         struct instruction_group *g;
11770                         uint32_t j;
11771
11772                         /* Continue when the current instruction is not a jump destination. */
11773                         if (!data->n_users)
11774                                 continue;
11775
11776                         g = instruction_group_list_group_find(igl, i);
11777                         if (!g)
11778                                 goto error;
11779
11780                         /* Find out all the jump instructions with this destination. */
11781                         for (j = 0; j < p->n_instructions; j++) {
11782                                 struct instruction *jmp_instr = &p->instructions[j];
11783                                 struct instruction_data *jmp_data = &p->instruction_data[j];
11784                                 struct instruction_group *jmp_g, *new_g;
11785
11786                                 /* Continue when not a jump instruction. Even when jump instruction,
11787                                  * continue when the jump destination is not this instruction.
11788                                  */
11789                                 if (!instruction_is_jmp(jmp_instr) ||
11790                                     strcmp(jmp_data->jmp_label, data->label))
11791                                         continue;
11792
11793                                 jmp_g = instruction_group_list_group_find(igl, j);
11794                                 if (!jmp_g)
11795                                         goto error;
11796
11797                                 /* Continue when both the jump instruction and the jump destination
11798                                  * instruction are in the same group. Even when in different groups,
11799                                  * still continue if the jump destination instruction is already the
11800                                  * first instruction of its group.
11801                                  */
11802                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11803                                         continue;
11804
11805                                 /* Split the group of the current jump destination instruction to
11806                                  * make this instruction the first instruction of a new group.
11807                                  */
11808                                 new_g = calloc(1, sizeof(struct instruction_group));
11809                                 if (!new_g)
11810                                         goto error;
11811
11812                                 new_g->group_id = n_groups;
11813                                 new_g->first_instr_id = i;
11814                                 new_g->last_instr_id = g->last_instr_id;
11815
11816                                 g->last_instr_id = i - 1;
11817
11818                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11819                                 n_groups++;
11820                                 is_modified = 1;
11821
11822                                 /* The decision to split this group (to make the current instruction
11823                                  * the first instruction of a new group) is already taken and fully
11824                                  * implemented, so no need to search for more reasons to do it.
11825                                  */
11826                                 break;
11827                         }
11828                 }
11829
11830                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11831                  * previously considered local (i.e. the jump destination is in the same group as
11832                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11833                  * different group than the jump instruction). Wost case scenario: each instruction
11834                  * that is a jump destination ends up as the first instruction of its group.
11835                  */
11836                 if (!is_modified)
11837                         break;
11838         }
11839
11840         /* Re-assign the group IDs to be in incremental order. */
11841         i = 0;
11842         TAILQ_FOREACH(g, igl, node) {
11843                 g->group_id = i;
11844
11845                 i++;
11846         }
11847
11848         return igl;
11849
11850 error:
11851         instruction_group_list_free(igl);
11852
11853         free(g);
11854
11855         return NULL;
11856 }
11857
11858 static void
11859 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11860                                uint32_t instr_pos,
11861                                struct instruction *instr,
11862                                FILE *f)
11863 {
11864         fprintf(f,
11865                 "%s(p, t, &pipeline_instructions[%u]);\n"
11866                 "\tthread_ip_reset(p, t);\n"
11867                 "\tinstr_rx_exec(p);\n"
11868                 "\treturn;\n",
11869                 instr_type_to_func(instr),
11870                 instr_pos);
11871 }
11872
11873 static int
11874 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11875                            struct instruction_group_list *igl,
11876                            uint32_t jmp_instr_id,
11877                            struct instruction *jmp_instr,
11878                            struct instruction_data *jmp_data,
11879                            FILE *f)
11880 {
11881         struct instruction_group *jmp_g, *g;
11882         struct instruction_data *data;
11883         uint32_t instr_id;
11884
11885         switch (jmp_instr->type) {
11886         case INSTR_JMP:
11887                 break;
11888
11889         case INSTR_JMP_VALID:
11890                 fprintf(f,
11891                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11892                         jmp_instr_id);
11893                 break;
11894
11895         case INSTR_JMP_INVALID:
11896                 fprintf(f,
11897                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11898                         jmp_instr_id);
11899                 break;
11900
11901         case INSTR_JMP_HIT:
11902                 fprintf(f,
11903                         "if (t->hit)\n");
11904                 break;
11905
11906         case INSTR_JMP_MISS:
11907                 fprintf(f,
11908                         "if (!t->hit)\n");
11909                 break;
11910
11911         case INSTR_JMP_ACTION_HIT:
11912                 fprintf(f,
11913                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
11914                         jmp_instr_id);
11915                 break;
11916
11917         case INSTR_JMP_ACTION_MISS:
11918                 fprintf(f,
11919                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
11920                         jmp_instr_id);
11921                 break;
11922
11923         case INSTR_JMP_EQ:
11924                 fprintf(f,
11925                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11926                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11927                         jmp_instr_id,
11928                         jmp_instr_id);
11929                 break;
11930
11931         case INSTR_JMP_EQ_MH:
11932                 fprintf(f,
11933                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11934                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11935                         jmp_instr_id,
11936                         jmp_instr_id);
11937                 break;
11938
11939         case INSTR_JMP_EQ_HM:
11940                 fprintf(f,
11941                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11942                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11943                         jmp_instr_id,
11944                         jmp_instr_id);
11945                 break;
11946
11947         case INSTR_JMP_EQ_HH:
11948                 fprintf(f,
11949                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11950                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11951                         jmp_instr_id,
11952                         jmp_instr_id);
11953                 break;
11954
11955         case INSTR_JMP_EQ_I:
11956                 fprintf(f,
11957                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11958                         "pipeline_instructions[%u].jmp.b_val)",
11959                         jmp_instr_id,
11960                         jmp_instr_id);
11961                 break;
11962
11963         case INSTR_JMP_NEQ:
11964                 fprintf(f,
11965                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11966                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11967                         jmp_instr_id,
11968                         jmp_instr_id);
11969                 break;
11970
11971         case INSTR_JMP_NEQ_MH:
11972                 fprintf(f,
11973                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11974                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11975                         jmp_instr_id,
11976                         jmp_instr_id);
11977                 break;
11978
11979         case INSTR_JMP_NEQ_HM:
11980                 fprintf(f,
11981                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11982                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11983                         jmp_instr_id,
11984                         jmp_instr_id);
11985                 break;
11986
11987         case INSTR_JMP_NEQ_HH:
11988                 fprintf(f,
11989                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11990                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11991                         jmp_instr_id,
11992                         jmp_instr_id);
11993                 break;
11994
11995         case INSTR_JMP_NEQ_I:
11996                 fprintf(f,
11997                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11998                         "pipeline_instructions[%u].jmp.b_val)",
11999                         jmp_instr_id,
12000                         jmp_instr_id);
12001                 break;
12002
12003         case INSTR_JMP_LT:
12004                 fprintf(f,
12005                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12006                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12007                         jmp_instr_id,
12008                         jmp_instr_id);
12009                 break;
12010
12011         case INSTR_JMP_LT_MH:
12012                 fprintf(f,
12013                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12014                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12015                         jmp_instr_id,
12016                         jmp_instr_id);
12017                 break;
12018
12019         case INSTR_JMP_LT_HM:
12020                 fprintf(f,
12021                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12022                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12023                         jmp_instr_id,
12024                         jmp_instr_id);
12025                 break;
12026
12027         case INSTR_JMP_LT_HH:
12028                 fprintf(f,
12029                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12030                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12031                         jmp_instr_id,
12032                         jmp_instr_id);
12033                 break;
12034
12035         case INSTR_JMP_LT_MI:
12036                 fprintf(f,
12037                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12038                         "pipeline_instructions[%u].jmp.b_val)",
12039                         jmp_instr_id,
12040                         jmp_instr_id);
12041                 break;
12042
12043         case INSTR_JMP_LT_HI:
12044                 fprintf(f,
12045                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12046                         "pipeline_instructions[%u].jmp.b_val)",
12047                         jmp_instr_id,
12048                         jmp_instr_id);
12049                 break;
12050
12051         case INSTR_JMP_GT:
12052                 fprintf(f,
12053                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12054                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12055                         jmp_instr_id,
12056                         jmp_instr_id);
12057                 break;
12058
12059         case INSTR_JMP_GT_MH:
12060                 fprintf(f,
12061                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12062                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12063                         jmp_instr_id,
12064                         jmp_instr_id);
12065                 break;
12066
12067         case INSTR_JMP_GT_HM:
12068                 fprintf(f,
12069                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12070                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12071                         jmp_instr_id,
12072                         jmp_instr_id);
12073                 break;
12074
12075         case INSTR_JMP_GT_HH:
12076                 fprintf(f,
12077                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12078                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12079                         jmp_instr_id,
12080                         jmp_instr_id);
12081                 break;
12082
12083         case INSTR_JMP_GT_MI:
12084                 fprintf(f,
12085                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12086                         "pipeline_instructions[%u].jmp.b_val)",
12087                         jmp_instr_id,
12088                         jmp_instr_id);
12089                 break;
12090
12091         case INSTR_JMP_GT_HI:
12092                 fprintf(f,
12093                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12094                         "pipeline_instructions[%u].jmp.b_val)",
12095                         jmp_instr_id,
12096                         jmp_instr_id);
12097                 break;
12098
12099         default:
12100                 break;
12101         }
12102
12103         /* Find the instruction group of the jump instruction. */
12104         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12105         if (!jmp_g)
12106                 return -EINVAL;
12107
12108         /* Find the instruction group of the jump destination instruction. */
12109         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12110         if (!data)
12111                 return -EINVAL;
12112
12113         instr_id = data - p->instruction_data;
12114
12115         g = instruction_group_list_group_find(igl, instr_id);
12116         if (!g)
12117                 return -EINVAL;
12118
12119         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12120          * instruction group).
12121          */
12122         if (g->group_id == jmp_g->group_id)
12123                 fprintf(f,
12124                         "\n\t\tgoto %s;\n",
12125                         jmp_data->jmp_label);
12126         else
12127                 fprintf(f,
12128                         " {\n"
12129                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12130                         "\t\treturn;\n"
12131                         "\t}\n\n",
12132                         g->group_id);
12133
12134         return 0;
12135 }
12136
12137 static void
12138 instruction_group_list_codegen(struct instruction_group_list *igl,
12139                                struct rte_swx_pipeline *p,
12140                                FILE *f)
12141 {
12142         struct instruction_group *g;
12143         uint32_t i;
12144         int is_required = 0;
12145
12146         /* Check if code generation is required. */
12147         TAILQ_FOREACH(g, igl, node)
12148                 if (g->first_instr_id < g->last_instr_id)
12149                         is_required = 1;
12150
12151         if (!is_required)
12152                 return;
12153
12154         /* Generate the code for the pipeline instruction array. */
12155         fprintf(f,
12156                 "static const struct instruction pipeline_instructions[] = {\n");
12157
12158         for (i = 0; i < p->n_instructions; i++) {
12159                 struct instruction *instr = &p->instructions[i];
12160                 instruction_export_t func = export_table[instr->type];
12161
12162                 func(instr, f);
12163         }
12164
12165         fprintf(f, "};\n\n");
12166
12167         /* Generate the code for the pipeline functions: one function for each instruction group
12168          * that contains more than one instruction.
12169          */
12170         TAILQ_FOREACH(g, igl, node) {
12171                 struct instruction *last_instr;
12172                 uint32_t j;
12173
12174                 /* Skip if group contains a single instruction. */
12175                 if (g->last_instr_id == g->first_instr_id)
12176                         continue;
12177
12178                 /* Generate new pipeline function. */
12179                 fprintf(f,
12180                         "void\n"
12181                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12182                         "{\n"
12183                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12184                         "\n",
12185                         g->group_id);
12186
12187                 /* Generate the code for each pipeline instruction. */
12188                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12189                         struct instruction *instr = &p->instructions[j];
12190                         struct instruction_data *data = &p->instruction_data[j];
12191
12192                         /* Label, if present. */
12193                         if (data->label[0])
12194                                 fprintf(f, "\n%s : ", data->label);
12195                         else
12196                                 fprintf(f, "\n\t");
12197
12198                         /* TX instruction type. */
12199                         if (instruction_does_tx(instr)) {
12200                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12201                                 continue;
12202                         }
12203
12204                         /* Jump instruction type. */
12205                         if (instruction_is_jmp(instr)) {
12206                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12207                                 continue;
12208                         }
12209
12210                         /* Any other instruction type. */
12211                         fprintf(f,
12212                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12213                                 instr_type_to_func(instr),
12214                                 j);
12215                 }
12216
12217                 /* Finalize the generated pipeline function. For some instructions such as TX,
12218                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12219                  * decided unconditionally and the instruction pointer of the current thread set
12220                  * accordingly; for all the other instructions, the instruction pointer must be
12221                  * incremented now.
12222                  */
12223                 last_instr = &p->instructions[g->last_instr_id];
12224
12225                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12226                         fprintf(f,
12227                                 "thread_ip_inc(p);\n");
12228
12229                 fprintf(f,
12230                         "}\n"
12231                         "\n");
12232         }
12233 }
12234
12235 static uint32_t
12236 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12237 {
12238         struct instruction_group *g;
12239         uint32_t n_custom_instr = 0;
12240
12241         /* Groups with a single instruction: no function is generated for this group, the group
12242          * keeps its current instruction. Groups with more than two instructions: one function and
12243          * the associated custom instruction get generated for each such group.
12244          */
12245         TAILQ_FOREACH(g, igl, node) {
12246                 if (g->first_instr_id == g->last_instr_id)
12247                         continue;
12248
12249                 n_custom_instr++;
12250         }
12251
12252         return n_custom_instr;
12253 }
12254
12255 static int
12256 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12257 {
12258         struct action *a;
12259         FILE *f = NULL;
12260
12261         /* Create the .c file. */
12262         f = fopen("/tmp/pipeline.c", "w");
12263         if (!f)
12264                 return -EIO;
12265
12266         /* Include the .h file. */
12267         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12268
12269         /* Add the code for each action. */
12270         TAILQ_FOREACH(a, &p->actions, node) {
12271                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12272
12273                 action_data_codegen(a, f);
12274
12275                 fprintf(f, "\n");
12276
12277                 action_instr_codegen(a, f);
12278
12279                 fprintf(f, "\n");
12280         }
12281
12282         /* Add the pipeline code. */
12283         instruction_group_list_codegen(igl, p, f);
12284
12285         /* Close the .c file. */
12286         fclose(f);
12287
12288         return 0;
12289 }
12290
12291 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12292 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12293 #endif
12294
12295 static int
12296 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12297 {
12298         struct action *a;
12299         struct instruction_group *g;
12300         char *dir_in, *buffer = NULL;
12301         const char *dir_out;
12302         int status = 0;
12303
12304         /* Get the environment variables. */
12305         dir_in = getenv("RTE_INSTALL_DIR");
12306         if (!dir_in) {
12307                 status = -EINVAL;
12308                 goto free;
12309         }
12310
12311         dir_out = "/tmp";
12312
12313         /* Memory allocation for the command buffer. */
12314         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12315         if (!buffer) {
12316                 status = -ENOMEM;
12317                 goto free;
12318         }
12319
12320         snprintf(buffer,
12321                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12322                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12323                  "-I %s/lib/pipeline "
12324                  "-I %s/lib/eal/include "
12325                  "-I %s/lib/eal/x86/include "
12326                  "-I %s/lib/eal/include/generic "
12327                  "-I %s/lib/meter "
12328                  "-I %s/lib/port "
12329                  "-I %s/lib/table "
12330                  "-I %s/lib/pipeline "
12331                  "-I %s/config "
12332                  "-I %s/build "
12333                  "-I %s/lib/eal/linux/include "
12334                  ">%s/pipeline.log 2>&1 "
12335                  "&& "
12336                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12337                  ">>%s/pipeline.log 2>&1",
12338                  dir_out,
12339                  dir_out,
12340                  dir_in,
12341                  dir_in,
12342                  dir_in,
12343                  dir_in,
12344                  dir_in,
12345                  dir_in,
12346                  dir_in,
12347                  dir_in,
12348                  dir_in,
12349                  dir_in,
12350                  dir_in,
12351                  dir_out,
12352                  dir_out,
12353                  dir_out,
12354                  dir_out);
12355
12356         /* Build the shared object library. */
12357         status = system(buffer);
12358         if (status)
12359                 goto free;
12360
12361         /* Open library. */
12362         snprintf(buffer,
12363                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12364                  "%s/libpipeline.so",
12365                  dir_out);
12366
12367         p->lib = dlopen(buffer, RTLD_LAZY);
12368         if (!p->lib) {
12369                 status = -EIO;
12370                 goto free;
12371         }
12372
12373         /* Get the action function symbols. */
12374         TAILQ_FOREACH(a, &p->actions, node) {
12375                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12376
12377                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12378                 if (!p->action_funcs[a->id]) {
12379                         status = -EINVAL;
12380                         goto free;
12381                 }
12382         }
12383
12384         /* Get the pipeline function symbols. */
12385         TAILQ_FOREACH(g, igl, node) {
12386                 if (g->first_instr_id == g->last_instr_id)
12387                         continue;
12388
12389                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12390
12391                 g->func = dlsym(p->lib, buffer);
12392                 if (!g->func) {
12393                         status = -EINVAL;
12394                         goto free;
12395                 }
12396         }
12397
12398 free:
12399         if (status && p->lib) {
12400                 dlclose(p->lib);
12401                 p->lib = NULL;
12402         }
12403
12404         free(buffer);
12405
12406         return status;
12407 }
12408
12409 static int
12410 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12411                       struct instruction_group_list *igl)
12412 {
12413         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12414
12415         /* Check that enough space is available within the pipeline instruction table to store all
12416          * the custom instructions.
12417          */
12418         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12419                 return -ENOSPC;
12420
12421         return 0;
12422 }
12423
12424 static void
12425 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12426 {
12427         struct instruction_group *g;
12428         uint32_t i;
12429
12430         /* Pipeline table instructions. */
12431         for (i = 0; i < p->n_instructions; i++) {
12432                 struct instruction *instr = &p->instructions[i];
12433
12434                 if (instr->type == INSTR_TABLE)
12435                         instr->type = INSTR_TABLE_AF;
12436
12437                 if (instr->type == INSTR_LEARNER)
12438                         instr->type = INSTR_LEARNER_AF;
12439         }
12440
12441         /* Pipeline custom instructions. */
12442         i = 0;
12443         TAILQ_FOREACH(g, igl, node) {
12444                 struct instruction *instr = &p->instructions[g->first_instr_id];
12445                 uint32_t j;
12446
12447                 if (g->first_instr_id == g->last_instr_id)
12448                         continue;
12449
12450                 /* Install a new custom instruction. */
12451                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12452
12453                 /* First instruction of the group: change its type to the new custom instruction. */
12454                 instr->type = INSTR_CUSTOM_0 + i;
12455
12456                 /* All the subsequent instructions of the group: invalidate. */
12457                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12458                         struct instruction_data *data = &p->instruction_data[j];
12459
12460                         data->invalid = 1;
12461                 }
12462
12463                 i++;
12464         }
12465
12466         /* Remove the invalidated instructions. */
12467         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12468
12469         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12470          * instructions that are the only instruction within their group, so they were left
12471          * unmodified).
12472          */
12473         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12474 }
12475
12476 static int
12477 pipeline_compile(struct rte_swx_pipeline *p)
12478 {
12479         struct instruction_group_list *igl = NULL;
12480         int status = 0;
12481
12482         igl = instruction_group_list_create(p);
12483         if (!igl) {
12484                 status = -ENOMEM;
12485                 goto free;
12486         }
12487
12488         /* Code generation. */
12489         status = pipeline_codegen(p, igl);
12490         if (status)
12491                 goto free;
12492
12493         /* Build and load the shared object library. */
12494         status = pipeline_libload(p, igl);
12495         if (status)
12496                 goto free;
12497
12498         /* Adjust instructions. */
12499         status = pipeline_adjust_check(p, igl);
12500         if (status)
12501                 goto free;
12502
12503         pipeline_adjust(p, igl);
12504
12505 free:
12506         instruction_group_list_free(igl);
12507
12508         return status;
12509 }