lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8 #include <dlfcn.h>
   9
  10 #include "rte_swx_pipeline_internal.h"
  11
  12 #define CHECK(condition, err_code)                                             \
  13 do {                                                                           \
  14         if (!(condition))                                                      \
  15                 return -(err_code);                                            \
  16 } while (0)
  17
  18 #define CHECK_NAME(name, err_code)                                             \
  19         CHECK((name) &&                                                        \
  20               (name)[0] &&                                                     \
  21               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  22               err_code)
  23
  24 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  25         CHECK((instr) &&                                                       \
  26               (instr)[0] &&                                                    \
  27               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  28                RTE_SWX_INSTRUCTION_SIZE),                                      \
  29               err_code)
  30
  31 /*
  32  * Environment.
  33  */
  34 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  35
  36 #include <rte_malloc.h>
  37
  38 static void *
  39 env_malloc(size_t size, size_t alignment, int numa_node)
  40 {
  41         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  42 }
  43
  44 static void
  45 env_free(void *start, size_t size __rte_unused)
  46 {
  47         rte_free(start);
  48 }
  49
  50 #else
  51
  52 #include <numa.h>
  53
  54 static void *
  55 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  56 {
  57         void *start;
  58
  59         if (numa_available() == -1)
  60                 return NULL;
  61
  62         start = numa_alloc_onnode(size, numa_node);
  63         if (!start)
  64                 return NULL;
  65
  66         memset(start, 0, size);
  67         return start;
  68 }
  69
  70 static void
  71 env_free(void *start, size_t size)
  72 {
  73         if (numa_available() == -1)
  74                 return;
  75
  76         numa_free(start, size);
  77 }
  78
  79 #endif
  80
  81 /*
  82  * Struct.
  83  */
  84 static struct struct_type *
  85 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  86 {
  87         struct struct_type *elem;
  88
  89         TAILQ_FOREACH(elem, &p->struct_types, node)
  90                 if (strcmp(elem->name, name) == 0)
  91                         return elem;
  92
  93         return NULL;
  94 }
  95
  96 static struct field *
  97 struct_type_field_find(struct struct_type *st, const char *name)
  98 {
  99         uint32_t i;
 100
 101         for (i = 0; i < st->n_fields; i++) {
 102                 struct field *f = &st->fields[i];
 103
 104                 if (strcmp(f->name, name) == 0)
 105                         return f;
 106         }
 107
 108         return NULL;
 109 }
 110
 111 int
 112 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 113                                       const char *name,
 114                                       struct rte_swx_field_params *fields,
 115                                       uint32_t n_fields,
 116                                       int last_field_has_variable_size)
 117 {
 118         struct struct_type *st;
 119         uint32_t i;
 120
 121         CHECK(p, EINVAL);
 122         CHECK_NAME(name, EINVAL);
 123         CHECK(fields, EINVAL);
 124         CHECK(n_fields, EINVAL);
 125
 126         for (i = 0; i < n_fields; i++) {
 127                 struct rte_swx_field_params *f = &fields[i];
 128                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 129                 uint32_t j;
 130
 131                 CHECK_NAME(f->name, EINVAL);
 132                 CHECK(f->n_bits, EINVAL);
 133                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 134                 CHECK((f->n_bits & 7) == 0, EINVAL);
 135
 136                 for (j = 0; j < i; j++) {
 137                         struct rte_swx_field_params *f_prev = &fields[j];
 138
 139                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 140                 }
 141         }
 142
 143         CHECK(!struct_type_find(p, name), EEXIST);
 144
 145         /* Node allocation. */
 146         st = calloc(1, sizeof(struct struct_type));
 147         CHECK(st, ENOMEM);
 148
 149         st->fields = calloc(n_fields, sizeof(struct field));
 150         if (!st->fields) {
 151                 free(st);
 152                 CHECK(0, ENOMEM);
 153         }
 154
 155         /* Node initialization. */
 156         strcpy(st->name, name);
 157         for (i = 0; i < n_fields; i++) {
 158                 struct field *dst = &st->fields[i];
 159                 struct rte_swx_field_params *src = &fields[i];
 160                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 161
 162                 strcpy(dst->name, src->name);
 163                 dst->n_bits = src->n_bits;
 164                 dst->offset = st->n_bits;
 165                 dst->var_size = var_size;
 166
 167                 st->n_bits += src->n_bits;
 168                 st->n_bits_min += var_size ? 0 : src->n_bits;
 169         }
 170         st->n_fields = n_fields;
 171         st->var_size = last_field_has_variable_size;
 172
 173         /* Node add to tailq. */
 174         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 175
 176         return 0;
 177 }
 178
 179 static int
 180 struct_build(struct rte_swx_pipeline *p)
 181 {
 182         uint32_t i;
 183
 184         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 185                 struct thread *t = &p->threads[i];
 186
 187                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 188                 CHECK(t->structs, ENOMEM);
 189         }
 190
 191         return 0;
 192 }
 193
 194 static void
 195 struct_build_free(struct rte_swx_pipeline *p)
 196 {
 197         uint32_t i;
 198
 199         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 200                 struct thread *t = &p->threads[i];
 201
 202                 free(t->structs);
 203                 t->structs = NULL;
 204         }
 205 }
 206
 207 static void
 208 struct_free(struct rte_swx_pipeline *p)
 209 {
 210         struct_build_free(p);
 211
 212         /* Struct types. */
 213         for ( ; ; ) {
 214                 struct struct_type *elem;
 215
 216                 elem = TAILQ_FIRST(&p->struct_types);
 217                 if (!elem)
 218                         break;
 219
 220                 TAILQ_REMOVE(&p->struct_types, elem, node);
 221                 free(elem->fields);
 222                 free(elem);
 223         }
 224 }
 225
 226 /*
 227  * Input port.
 228  */
 229 static struct port_in_type *
 230 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 231 {
 232         struct port_in_type *elem;
 233
 234         if (!name)
 235                 return NULL;
 236
 237         TAILQ_FOREACH(elem, &p->port_in_types, node)
 238                 if (strcmp(elem->name, name) == 0)
 239                         return elem;
 240
 241         return NULL;
 242 }
 243
 244 int
 245 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 246                                        const char *name,
 247                                        struct rte_swx_port_in_ops *ops)
 248 {
 249         struct port_in_type *elem;
 250
 251         CHECK(p, EINVAL);
 252         CHECK_NAME(name, EINVAL);
 253         CHECK(ops, EINVAL);
 254         CHECK(ops->create, EINVAL);
 255         CHECK(ops->free, EINVAL);
 256         CHECK(ops->pkt_rx, EINVAL);
 257         CHECK(ops->stats_read, EINVAL);
 258
 259         CHECK(!port_in_type_find(p, name), EEXIST);
 260
 261         /* Node allocation. */
 262         elem = calloc(1, sizeof(struct port_in_type));
 263         CHECK(elem, ENOMEM);
 264
 265         /* Node initialization. */
 266         strcpy(elem->name, name);
 267         memcpy(&elem->ops, ops, sizeof(*ops));
 268
 269         /* Node add to tailq. */
 270         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 271
 272         return 0;
 273 }
 274
 275 static struct port_in *
 276 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 277 {
 278         struct port_in *port;
 279
 280         TAILQ_FOREACH(port, &p->ports_in, node)
 281                 if (port->id == port_id)
 282                         return port;
 283
 284         return NULL;
 285 }
 286
 287 int
 288 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 289                                 uint32_t port_id,
 290                                 const char *port_type_name,
 291                                 void *args)
 292 {
 293         struct port_in_type *type = NULL;
 294         struct port_in *port = NULL;
 295         void *obj = NULL;
 296
 297         CHECK(p, EINVAL);
 298
 299         CHECK(!port_in_find(p, port_id), EINVAL);
 300
 301         CHECK_NAME(port_type_name, EINVAL);
 302         type = port_in_type_find(p, port_type_name);
 303         CHECK(type, EINVAL);
 304
 305         obj = type->ops.create(args);
 306         CHECK(obj, ENODEV);
 307
 308         /* Node allocation. */
 309         port = calloc(1, sizeof(struct port_in));
 310         CHECK(port, ENOMEM);
 311
 312         /* Node initialization. */
 313         port->type = type;
 314         port->obj = obj;
 315         port->id = port_id;
 316
 317         /* Node add to tailq. */
 318         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 319         if (p->n_ports_in < port_id + 1)
 320                 p->n_ports_in = port_id + 1;
 321
 322         return 0;
 323 }
 324
 325 static int
 326 port_in_build(struct rte_swx_pipeline *p)
 327 {
 328         struct port_in *port;
 329         uint32_t i;
 330
 331         CHECK(p->n_ports_in, EINVAL);
 332         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 333
 334         for (i = 0; i < p->n_ports_in; i++)
 335                 CHECK(port_in_find(p, i), EINVAL);
 336
 337         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 338         CHECK(p->in, ENOMEM);
 339
 340         TAILQ_FOREACH(port, &p->ports_in, node) {
 341                 struct port_in_runtime *in = &p->in[port->id];
 342
 343                 in->pkt_rx = port->type->ops.pkt_rx;
 344                 in->obj = port->obj;
 345         }
 346
 347         return 0;
 348 }
 349
 350 static void
 351 port_in_build_free(struct rte_swx_pipeline *p)
 352 {
 353         free(p->in);
 354         p->in = NULL;
 355 }
 356
 357 static void
 358 port_in_free(struct rte_swx_pipeline *p)
 359 {
 360         port_in_build_free(p);
 361
 362         /* Input ports. */
 363         for ( ; ; ) {
 364                 struct port_in *port;
 365
 366                 port = TAILQ_FIRST(&p->ports_in);
 367                 if (!port)
 368                         break;
 369
 370                 TAILQ_REMOVE(&p->ports_in, port, node);
 371                 port->type->ops.free(port->obj);
 372                 free(port);
 373         }
 374
 375         /* Input port types. */
 376         for ( ; ; ) {
 377                 struct port_in_type *elem;
 378
 379                 elem = TAILQ_FIRST(&p->port_in_types);
 380                 if (!elem)
 381                         break;
 382
 383                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 384                 free(elem);
 385         }
 386 }
 387
 388 /*
 389  * Output port.
 390  */
 391 static struct port_out_type *
 392 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 393 {
 394         struct port_out_type *elem;
 395
 396         if (!name)
 397                 return NULL;
 398
 399         TAILQ_FOREACH(elem, &p->port_out_types, node)
 400                 if (!strcmp(elem->name, name))
 401                         return elem;
 402
 403         return NULL;
 404 }
 405
 406 int
 407 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 408                                         const char *name,
 409                                         struct rte_swx_port_out_ops *ops)
 410 {
 411         struct port_out_type *elem;
 412
 413         CHECK(p, EINVAL);
 414         CHECK_NAME(name, EINVAL);
 415         CHECK(ops, EINVAL);
 416         CHECK(ops->create, EINVAL);
 417         CHECK(ops->free, EINVAL);
 418         CHECK(ops->pkt_tx, EINVAL);
 419         CHECK(ops->stats_read, EINVAL);
 420
 421         CHECK(!port_out_type_find(p, name), EEXIST);
 422
 423         /* Node allocation. */
 424         elem = calloc(1, sizeof(struct port_out_type));
 425         CHECK(elem, ENOMEM);
 426
 427         /* Node initialization. */
 428         strcpy(elem->name, name);
 429         memcpy(&elem->ops, ops, sizeof(*ops));
 430
 431         /* Node add to tailq. */
 432         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 433
 434         return 0;
 435 }
 436
 437 static struct port_out *
 438 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 439 {
 440         struct port_out *port;
 441
 442         TAILQ_FOREACH(port, &p->ports_out, node)
 443                 if (port->id == port_id)
 444                         return port;
 445
 446         return NULL;
 447 }
 448
 449 int
 450 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 451                                  uint32_t port_id,
 452                                  const char *port_type_name,
 453                                  void *args)
 454 {
 455         struct port_out_type *type = NULL;
 456         struct port_out *port = NULL;
 457         void *obj = NULL;
 458
 459         CHECK(p, EINVAL);
 460
 461         CHECK(!port_out_find(p, port_id), EINVAL);
 462
 463         CHECK_NAME(port_type_name, EINVAL);
 464         type = port_out_type_find(p, port_type_name);
 465         CHECK(type, EINVAL);
 466
 467         obj = type->ops.create(args);
 468         CHECK(obj, ENODEV);
 469
 470         /* Node allocation. */
 471         port = calloc(1, sizeof(struct port_out));
 472         CHECK(port, ENOMEM);
 473
 474         /* Node initialization. */
 475         port->type = type;
 476         port->obj = obj;
 477         port->id = port_id;
 478
 479         /* Node add to tailq. */
 480         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 481         if (p->n_ports_out < port_id + 1)
 482                 p->n_ports_out = port_id + 1;
 483
 484         return 0;
 485 }
 486
 487 static int
 488 port_out_build(struct rte_swx_pipeline *p)
 489 {
 490         struct port_out *port;
 491         uint32_t i;
 492
 493         CHECK(p->n_ports_out, EINVAL);
 494
 495         for (i = 0; i < p->n_ports_out; i++)
 496                 CHECK(port_out_find(p, i), EINVAL);
 497
 498         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 499         CHECK(p->out, ENOMEM);
 500
 501         TAILQ_FOREACH(port, &p->ports_out, node) {
 502                 struct port_out_runtime *out = &p->out[port->id];
 503
 504                 out->pkt_tx = port->type->ops.pkt_tx;
 505                 out->flush = port->type->ops.flush;
 506                 out->obj = port->obj;
 507         }
 508
 509         return 0;
 510 }
 511
 512 static void
 513 port_out_build_free(struct rte_swx_pipeline *p)
 514 {
 515         free(p->out);
 516         p->out = NULL;
 517 }
 518
 519 static void
 520 port_out_free(struct rte_swx_pipeline *p)
 521 {
 522         port_out_build_free(p);
 523
 524         /* Output ports. */
 525         for ( ; ; ) {
 526                 struct port_out *port;
 527
 528                 port = TAILQ_FIRST(&p->ports_out);
 529                 if (!port)
 530                         break;
 531
 532                 TAILQ_REMOVE(&p->ports_out, port, node);
 533                 port->type->ops.free(port->obj);
 534                 free(port);
 535         }
 536
 537         /* Output port types. */
 538         for ( ; ; ) {
 539                 struct port_out_type *elem;
 540
 541                 elem = TAILQ_FIRST(&p->port_out_types);
 542                 if (!elem)
 543                         break;
 544
 545                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 546                 free(elem);
 547         }
 548 }
 549
 550 /*
 551  * Extern object.
 552  */
 553 static struct extern_type *
 554 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 555 {
 556         struct extern_type *elem;
 557
 558         TAILQ_FOREACH(elem, &p->extern_types, node)
 559                 if (strcmp(elem->name, name) == 0)
 560                         return elem;
 561
 562         return NULL;
 563 }
 564
 565 static struct extern_type_member_func *
 566 extern_type_member_func_find(struct extern_type *type, const char *name)
 567 {
 568         struct extern_type_member_func *elem;
 569
 570         TAILQ_FOREACH(elem, &type->funcs, node)
 571                 if (strcmp(elem->name, name) == 0)
 572                         return elem;
 573
 574         return NULL;
 575 }
 576
 577 static struct extern_obj *
 578 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 579 {
 580         struct extern_obj *elem;
 581
 582         TAILQ_FOREACH(elem, &p->extern_objs, node)
 583                 if (strcmp(elem->name, name) == 0)
 584                         return elem;
 585
 586         return NULL;
 587 }
 588
 589 static struct extern_type_member_func *
 590 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 591                              const char *name,
 592                              struct extern_obj **obj)
 593 {
 594         struct extern_obj *object;
 595         struct extern_type_member_func *func;
 596         char *object_name, *func_name;
 597
 598         if (name[0] != 'e' || name[1] != '.')
 599                 return NULL;
 600
 601         object_name = strdup(&name[2]);
 602         if (!object_name)
 603                 return NULL;
 604
 605         func_name = strchr(object_name, '.');
 606         if (!func_name) {
 607                 free(object_name);
 608                 return NULL;
 609         }
 610
 611         *func_name = 0;
 612         func_name++;
 613
 614         object = extern_obj_find(p, object_name);
 615         if (!object) {
 616                 free(object_name);
 617                 return NULL;
 618         }
 619
 620         func = extern_type_member_func_find(object->type, func_name);
 621         if (!func) {
 622                 free(object_name);
 623                 return NULL;
 624         }
 625
 626         if (obj)
 627                 *obj = object;
 628
 629         free(object_name);
 630         return func;
 631 }
 632
 633 static struct field *
 634 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 635                                const char *name,
 636                                struct extern_obj **object)
 637 {
 638         struct extern_obj *obj;
 639         struct field *f;
 640         char *obj_name, *field_name;
 641
 642         if ((name[0] != 'e') || (name[1] != '.'))
 643                 return NULL;
 644
 645         obj_name = strdup(&name[2]);
 646         if (!obj_name)
 647                 return NULL;
 648
 649         field_name = strchr(obj_name, '.');
 650         if (!field_name) {
 651                 free(obj_name);
 652                 return NULL;
 653         }
 654
 655         *field_name = 0;
 656         field_name++;
 657
 658         obj = extern_obj_find(p, obj_name);
 659         if (!obj) {
 660                 free(obj_name);
 661                 return NULL;
 662         }
 663
 664         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 665         if (!f) {
 666                 free(obj_name);
 667                 return NULL;
 668         }
 669
 670         if (object)
 671                 *object = obj;
 672
 673         free(obj_name);
 674         return f;
 675 }
 676
 677 int
 678 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 679         const char *name,
 680         const char *mailbox_struct_type_name,
 681         rte_swx_extern_type_constructor_t constructor,
 682         rte_swx_extern_type_destructor_t destructor)
 683 {
 684         struct extern_type *elem;
 685         struct struct_type *mailbox_struct_type;
 686
 687         CHECK(p, EINVAL);
 688
 689         CHECK_NAME(name, EINVAL);
 690         CHECK(!extern_type_find(p, name), EEXIST);
 691
 692         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 693         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 694         CHECK(mailbox_struct_type, EINVAL);
 695         CHECK(!mailbox_struct_type->var_size, EINVAL);
 696
 697         CHECK(constructor, EINVAL);
 698         CHECK(destructor, EINVAL);
 699
 700         /* Node allocation. */
 701         elem = calloc(1, sizeof(struct extern_type));
 702         CHECK(elem, ENOMEM);
 703
 704         /* Node initialization. */
 705         strcpy(elem->name, name);
 706         elem->mailbox_struct_type = mailbox_struct_type;
 707         elem->constructor = constructor;
 708         elem->destructor = destructor;
 709         TAILQ_INIT(&elem->funcs);
 710
 711         /* Node add to tailq. */
 712         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 713
 714         return 0;
 715 }
 716
 717 int
 718 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 719         const char *extern_type_name,
 720         const char *name,
 721         rte_swx_extern_type_member_func_t member_func)
 722 {
 723         struct extern_type *type;
 724         struct extern_type_member_func *type_member;
 725
 726         CHECK(p, EINVAL);
 727
 728         CHECK_NAME(extern_type_name, EINVAL);
 729         type = extern_type_find(p, extern_type_name);
 730         CHECK(type, EINVAL);
 731         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 732
 733         CHECK_NAME(name, EINVAL);
 734         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 735
 736         CHECK(member_func, EINVAL);
 737
 738         /* Node allocation. */
 739         type_member = calloc(1, sizeof(struct extern_type_member_func));
 740         CHECK(type_member, ENOMEM);
 741
 742         /* Node initialization. */
 743         strcpy(type_member->name, name);
 744         type_member->func = member_func;
 745         type_member->id = type->n_funcs;
 746
 747         /* Node add to tailq. */
 748         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 749         type->n_funcs++;
 750
 751         return 0;
 752 }
 753
 754 int
 755 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 756                                       const char *extern_type_name,
 757                                       const char *name,
 758                                       const char *args)
 759 {
 760         struct extern_type *type;
 761         struct extern_obj *obj;
 762         void *obj_handle;
 763
 764         CHECK(p, EINVAL);
 765
 766         CHECK_NAME(extern_type_name, EINVAL);
 767         type = extern_type_find(p, extern_type_name);
 768         CHECK(type, EINVAL);
 769
 770         CHECK_NAME(name, EINVAL);
 771         CHECK(!extern_obj_find(p, name), EEXIST);
 772
 773         /* Node allocation. */
 774         obj = calloc(1, sizeof(struct extern_obj));
 775         CHECK(obj, ENOMEM);
 776
 777         /* Object construction. */
 778         obj_handle = type->constructor(args);
 779         if (!obj_handle) {
 780                 free(obj);
 781                 CHECK(0, ENODEV);
 782         }
 783
 784         /* Node initialization. */
 785         strcpy(obj->name, name);
 786         obj->type = type;
 787         obj->obj = obj_handle;
 788         obj->struct_id = p->n_structs;
 789         obj->id = p->n_extern_objs;
 790
 791         /* Node add to tailq. */
 792         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 793         p->n_extern_objs++;
 794         p->n_structs++;
 795
 796         return 0;
 797 }
 798
 799 static int
 800 extern_obj_build(struct rte_swx_pipeline *p)
 801 {
 802         uint32_t i;
 803
 804         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 805                 struct thread *t = &p->threads[i];
 806                 struct extern_obj *obj;
 807
 808                 t->extern_objs = calloc(p->n_extern_objs,
 809                                         sizeof(struct extern_obj_runtime));
 810                 CHECK(t->extern_objs, ENOMEM);
 811
 812                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 813                         struct extern_obj_runtime *r =
 814                                 &t->extern_objs[obj->id];
 815                         struct extern_type_member_func *func;
 816                         uint32_t mailbox_size =
 817                                 obj->type->mailbox_struct_type->n_bits / 8;
 818
 819                         r->obj = obj->obj;
 820
 821                         r->mailbox = calloc(1, mailbox_size);
 822                         CHECK(r->mailbox, ENOMEM);
 823
 824                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 825                                 r->funcs[func->id] = func->func;
 826
 827                         t->structs[obj->struct_id] = r->mailbox;
 828                 }
 829         }
 830
 831         return 0;
 832 }
 833
 834 static void
 835 extern_obj_build_free(struct rte_swx_pipeline *p)
 836 {
 837         uint32_t i;
 838
 839         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 840                 struct thread *t = &p->threads[i];
 841                 uint32_t j;
 842
 843                 if (!t->extern_objs)
 844                         continue;
 845
 846                 for (j = 0; j < p->n_extern_objs; j++) {
 847                         struct extern_obj_runtime *r = &t->extern_objs[j];
 848
 849                         free(r->mailbox);
 850                 }
 851
 852                 free(t->extern_objs);
 853                 t->extern_objs = NULL;
 854         }
 855 }
 856
 857 static void
 858 extern_obj_free(struct rte_swx_pipeline *p)
 859 {
 860         extern_obj_build_free(p);
 861
 862         /* Extern objects. */
 863         for ( ; ; ) {
 864                 struct extern_obj *elem;
 865
 866                 elem = TAILQ_FIRST(&p->extern_objs);
 867                 if (!elem)
 868                         break;
 869
 870                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 871                 if (elem->obj)
 872                         elem->type->destructor(elem->obj);
 873                 free(elem);
 874         }
 875
 876         /* Extern types. */
 877         for ( ; ; ) {
 878                 struct extern_type *elem;
 879
 880                 elem = TAILQ_FIRST(&p->extern_types);
 881                 if (!elem)
 882                         break;
 883
 884                 TAILQ_REMOVE(&p->extern_types, elem, node);
 885
 886                 for ( ; ; ) {
 887                         struct extern_type_member_func *func;
 888
 889                         func = TAILQ_FIRST(&elem->funcs);
 890                         if (!func)
 891                                 break;
 892
 893                         TAILQ_REMOVE(&elem->funcs, func, node);
 894                         free(func);
 895                 }
 896
 897                 free(elem);
 898         }
 899 }
 900
 901 /*
 902  * Extern function.
 903  */
 904 static struct extern_func *
 905 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 906 {
 907         struct extern_func *elem;
 908
 909         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 910                 if (strcmp(elem->name, name) == 0)
 911                         return elem;
 912
 913         return NULL;
 914 }
 915
 916 static struct extern_func *
 917 extern_func_parse(struct rte_swx_pipeline *p,
 918                   const char *name)
 919 {
 920         if (name[0] != 'f' || name[1] != '.')
 921                 return NULL;
 922
 923         return extern_func_find(p, &name[2]);
 924 }
 925
 926 static struct field *
 927 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 928                                 const char *name,
 929                                 struct extern_func **function)
 930 {
 931         struct extern_func *func;
 932         struct field *f;
 933         char *func_name, *field_name;
 934
 935         if ((name[0] != 'f') || (name[1] != '.'))
 936                 return NULL;
 937
 938         func_name = strdup(&name[2]);
 939         if (!func_name)
 940                 return NULL;
 941
 942         field_name = strchr(func_name, '.');
 943         if (!field_name) {
 944                 free(func_name);
 945                 return NULL;
 946         }
 947
 948         *field_name = 0;
 949         field_name++;
 950
 951         func = extern_func_find(p, func_name);
 952         if (!func) {
 953                 free(func_name);
 954                 return NULL;
 955         }
 956
 957         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 958         if (!f) {
 959                 free(func_name);
 960                 return NULL;
 961         }
 962
 963         if (function)
 964                 *function = func;
 965
 966         free(func_name);
 967         return f;
 968 }
 969
 970 int
 971 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 972                                       const char *name,
 973                                       const char *mailbox_struct_type_name,
 974                                       rte_swx_extern_func_t func)
 975 {
 976         struct extern_func *f;
 977         struct struct_type *mailbox_struct_type;
 978
 979         CHECK(p, EINVAL);
 980
 981         CHECK_NAME(name, EINVAL);
 982         CHECK(!extern_func_find(p, name), EEXIST);
 983
 984         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 985         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 986         CHECK(mailbox_struct_type, EINVAL);
 987         CHECK(!mailbox_struct_type->var_size, EINVAL);
 988
 989         CHECK(func, EINVAL);
 990
 991         /* Node allocation. */
 992         f = calloc(1, sizeof(struct extern_func));
 993         CHECK(func, ENOMEM);
 994
 995         /* Node initialization. */
 996         strcpy(f->name, name);
 997         f->mailbox_struct_type = mailbox_struct_type;
 998         f->func = func;
 999         f->struct_id = p->n_structs;
1000         f->id = p->n_extern_funcs;
1001
1002         /* Node add to tailq. */
1003         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1004         p->n_extern_funcs++;
1005         p->n_structs++;
1006
1007         return 0;
1008 }
1009
1010 static int
1011 extern_func_build(struct rte_swx_pipeline *p)
1012 {
1013         uint32_t i;
1014
1015         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1016                 struct thread *t = &p->threads[i];
1017                 struct extern_func *func;
1018
1019                 /* Memory allocation. */
1020                 t->extern_funcs = calloc(p->n_extern_funcs,
1021                                          sizeof(struct extern_func_runtime));
1022                 CHECK(t->extern_funcs, ENOMEM);
1023
1024                 /* Extern function. */
1025                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1026                         struct extern_func_runtime *r =
1027                                 &t->extern_funcs[func->id];
1028                         uint32_t mailbox_size =
1029                                 func->mailbox_struct_type->n_bits / 8;
1030
1031                         r->func = func->func;
1032
1033                         r->mailbox = calloc(1, mailbox_size);
1034                         CHECK(r->mailbox, ENOMEM);
1035
1036                         t->structs[func->struct_id] = r->mailbox;
1037                 }
1038         }
1039
1040         return 0;
1041 }
1042
1043 static void
1044 extern_func_build_free(struct rte_swx_pipeline *p)
1045 {
1046         uint32_t i;
1047
1048         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1049                 struct thread *t = &p->threads[i];
1050                 uint32_t j;
1051
1052                 if (!t->extern_funcs)
1053                         continue;
1054
1055                 for (j = 0; j < p->n_extern_funcs; j++) {
1056                         struct extern_func_runtime *r = &t->extern_funcs[j];
1057
1058                         free(r->mailbox);
1059                 }
1060
1061                 free(t->extern_funcs);
1062                 t->extern_funcs = NULL;
1063         }
1064 }
1065
1066 static void
1067 extern_func_free(struct rte_swx_pipeline *p)
1068 {
1069         extern_func_build_free(p);
1070
1071         for ( ; ; ) {
1072                 struct extern_func *elem;
1073
1074                 elem = TAILQ_FIRST(&p->extern_funcs);
1075                 if (!elem)
1076                         break;
1077
1078                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1079                 free(elem);
1080         }
1081 }
1082
1083 /*
1084  * Header.
1085  */
1086 static struct header *
1087 header_find(struct rte_swx_pipeline *p, const char *name)
1088 {
1089         struct header *elem;
1090
1091         TAILQ_FOREACH(elem, &p->headers, node)
1092                 if (strcmp(elem->name, name) == 0)
1093                         return elem;
1094
1095         return NULL;
1096 }
1097
1098 static struct header *
1099 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1100 {
1101         struct header *elem;
1102
1103         TAILQ_FOREACH(elem, &p->headers, node)
1104                 if (elem->struct_id == struct_id)
1105                         return elem;
1106
1107         return NULL;
1108 }
1109
1110 static struct header *
1111 header_parse(struct rte_swx_pipeline *p,
1112              const char *name)
1113 {
1114         if (name[0] != 'h' || name[1] != '.')
1115                 return NULL;
1116
1117         return header_find(p, &name[2]);
1118 }
1119
1120 static struct field *
1121 header_field_parse(struct rte_swx_pipeline *p,
1122                    const char *name,
1123                    struct header **header)
1124 {
1125         struct header *h;
1126         struct field *f;
1127         char *header_name, *field_name;
1128
1129         if ((name[0] != 'h') || (name[1] != '.'))
1130                 return NULL;
1131
1132         header_name = strdup(&name[2]);
1133         if (!header_name)
1134                 return NULL;
1135
1136         field_name = strchr(header_name, '.');
1137         if (!field_name) {
1138                 free(header_name);
1139                 return NULL;
1140         }
1141
1142         *field_name = 0;
1143         field_name++;
1144
1145         h = header_find(p, header_name);
1146         if (!h) {
1147                 free(header_name);
1148                 return NULL;
1149         }
1150
1151         f = struct_type_field_find(h->st, field_name);
1152         if (!f) {
1153                 free(header_name);
1154                 return NULL;
1155         }
1156
1157         if (header)
1158                 *header = h;
1159
1160         free(header_name);
1161         return f;
1162 }
1163
1164 int
1165 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1166                                         const char *name,
1167                                         const char *struct_type_name)
1168 {
1169         struct struct_type *st;
1170         struct header *h;
1171         size_t n_headers_max;
1172
1173         CHECK(p, EINVAL);
1174         CHECK_NAME(name, EINVAL);
1175         CHECK_NAME(struct_type_name, EINVAL);
1176
1177         CHECK(!header_find(p, name), EEXIST);
1178
1179         st = struct_type_find(p, struct_type_name);
1180         CHECK(st, EINVAL);
1181
1182         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1183         CHECK(p->n_headers < n_headers_max, ENOSPC);
1184
1185         /* Node allocation. */
1186         h = calloc(1, sizeof(struct header));
1187         CHECK(h, ENOMEM);
1188
1189         /* Node initialization. */
1190         strcpy(h->name, name);
1191         h->st = st;
1192         h->struct_id = p->n_structs;
1193         h->id = p->n_headers;
1194
1195         /* Node add to tailq. */
1196         TAILQ_INSERT_TAIL(&p->headers, h, node);
1197         p->n_headers++;
1198         p->n_structs++;
1199
1200         return 0;
1201 }
1202
1203 static int
1204 header_build(struct rte_swx_pipeline *p)
1205 {
1206         struct header *h;
1207         uint32_t n_bytes = 0, i;
1208
1209         TAILQ_FOREACH(h, &p->headers, node) {
1210                 n_bytes += h->st->n_bits / 8;
1211         }
1212
1213         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1214                 struct thread *t = &p->threads[i];
1215                 uint32_t offset = 0;
1216
1217                 t->headers = calloc(p->n_headers,
1218                                     sizeof(struct header_runtime));
1219                 CHECK(t->headers, ENOMEM);
1220
1221                 t->headers_out = calloc(p->n_headers,
1222                                         sizeof(struct header_out_runtime));
1223                 CHECK(t->headers_out, ENOMEM);
1224
1225                 t->header_storage = calloc(1, n_bytes);
1226                 CHECK(t->header_storage, ENOMEM);
1227
1228                 t->header_out_storage = calloc(1, n_bytes);
1229                 CHECK(t->header_out_storage, ENOMEM);
1230
1231                 TAILQ_FOREACH(h, &p->headers, node) {
1232                         uint8_t *header_storage;
1233                         uint32_t n_bytes =  h->st->n_bits / 8;
1234
1235                         header_storage = &t->header_storage[offset];
1236                         offset += n_bytes;
1237
1238                         t->headers[h->id].ptr0 = header_storage;
1239                         t->headers[h->id].n_bytes = n_bytes;
1240
1241                         t->structs[h->struct_id] = header_storage;
1242                 }
1243         }
1244
1245         return 0;
1246 }
1247
1248 static void
1249 header_build_free(struct rte_swx_pipeline *p)
1250 {
1251         uint32_t i;
1252
1253         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1254                 struct thread *t = &p->threads[i];
1255
1256                 free(t->headers_out);
1257                 t->headers_out = NULL;
1258
1259                 free(t->headers);
1260                 t->headers = NULL;
1261
1262                 free(t->header_out_storage);
1263                 t->header_out_storage = NULL;
1264
1265                 free(t->header_storage);
1266                 t->header_storage = NULL;
1267         }
1268 }
1269
1270 static void
1271 header_free(struct rte_swx_pipeline *p)
1272 {
1273         header_build_free(p);
1274
1275         for ( ; ; ) {
1276                 struct header *elem;
1277
1278                 elem = TAILQ_FIRST(&p->headers);
1279                 if (!elem)
1280                         break;
1281
1282                 TAILQ_REMOVE(&p->headers, elem, node);
1283                 free(elem);
1284         }
1285 }
1286
1287 /*
1288  * Meta-data.
1289  */
1290 static struct field *
1291 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1292 {
1293         if (!p->metadata_st)
1294                 return NULL;
1295
1296         if (name[0] != 'm' || name[1] != '.')
1297                 return NULL;
1298
1299         return struct_type_field_find(p->metadata_st, &name[2]);
1300 }
1301
1302 int
1303 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1304                                           const char *struct_type_name)
1305 {
1306         struct struct_type *st = NULL;
1307
1308         CHECK(p, EINVAL);
1309
1310         CHECK_NAME(struct_type_name, EINVAL);
1311         st  = struct_type_find(p, struct_type_name);
1312         CHECK(st, EINVAL);
1313         CHECK(!st->var_size, EINVAL);
1314         CHECK(!p->metadata_st, EINVAL);
1315
1316         p->metadata_st = st;
1317         p->metadata_struct_id = p->n_structs;
1318
1319         p->n_structs++;
1320
1321         return 0;
1322 }
1323
1324 static int
1325 metadata_build(struct rte_swx_pipeline *p)
1326 {
1327         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1328         uint32_t i;
1329
1330         /* Thread-level initialization. */
1331         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1332                 struct thread *t = &p->threads[i];
1333                 uint8_t *metadata;
1334
1335                 metadata = calloc(1, n_bytes);
1336                 CHECK(metadata, ENOMEM);
1337
1338                 t->metadata = metadata;
1339                 t->structs[p->metadata_struct_id] = metadata;
1340         }
1341
1342         return 0;
1343 }
1344
1345 static void
1346 metadata_build_free(struct rte_swx_pipeline *p)
1347 {
1348         uint32_t i;
1349
1350         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1351                 struct thread *t = &p->threads[i];
1352
1353                 free(t->metadata);
1354                 t->metadata = NULL;
1355         }
1356 }
1357
1358 static void
1359 metadata_free(struct rte_swx_pipeline *p)
1360 {
1361         metadata_build_free(p);
1362 }
1363
1364 /*
1365  * Instruction.
1366  */
1367 static int
1368 instruction_is_tx(enum instruction_type type)
1369 {
1370         switch (type) {
1371         case INSTR_TX:
1372         case INSTR_TX_I:
1373                 return 1;
1374
1375         default:
1376                 return 0;
1377         }
1378 }
1379
1380 static int
1381 instruction_does_tx(struct instruction *instr)
1382 {
1383         switch (instr->type) {
1384         case INSTR_TX:
1385         case INSTR_TX_I:
1386         case INSTR_HDR_EMIT_TX:
1387         case INSTR_HDR_EMIT2_TX:
1388         case INSTR_HDR_EMIT3_TX:
1389         case INSTR_HDR_EMIT4_TX:
1390         case INSTR_HDR_EMIT5_TX:
1391         case INSTR_HDR_EMIT6_TX:
1392         case INSTR_HDR_EMIT7_TX:
1393         case INSTR_HDR_EMIT8_TX:
1394                 return 1;
1395         default:
1396                 return 0;
1397         }
1398 }
1399
1400 static int
1401 instruction_is_jmp(struct instruction *instr)
1402 {
1403         switch (instr->type) {
1404         case INSTR_JMP:
1405         case INSTR_JMP_VALID:
1406         case INSTR_JMP_INVALID:
1407         case INSTR_JMP_HIT:
1408         case INSTR_JMP_MISS:
1409         case INSTR_JMP_ACTION_HIT:
1410         case INSTR_JMP_ACTION_MISS:
1411         case INSTR_JMP_EQ:
1412         case INSTR_JMP_EQ_MH:
1413         case INSTR_JMP_EQ_HM:
1414         case INSTR_JMP_EQ_HH:
1415         case INSTR_JMP_EQ_I:
1416         case INSTR_JMP_NEQ:
1417         case INSTR_JMP_NEQ_MH:
1418         case INSTR_JMP_NEQ_HM:
1419         case INSTR_JMP_NEQ_HH:
1420         case INSTR_JMP_NEQ_I:
1421         case INSTR_JMP_LT:
1422         case INSTR_JMP_LT_MH:
1423         case INSTR_JMP_LT_HM:
1424         case INSTR_JMP_LT_HH:
1425         case INSTR_JMP_LT_MI:
1426         case INSTR_JMP_LT_HI:
1427         case INSTR_JMP_GT:
1428         case INSTR_JMP_GT_MH:
1429         case INSTR_JMP_GT_HM:
1430         case INSTR_JMP_GT_HH:
1431         case INSTR_JMP_GT_MI:
1432         case INSTR_JMP_GT_HI:
1433                 return 1;
1434
1435         default:
1436                 return 0;
1437         }
1438 }
1439
1440 static int
1441 instruction_does_thread_yield(struct instruction *instr)
1442 {
1443         switch (instr->type) {
1444         case INSTR_RX:
1445         case INSTR_TABLE:
1446         case INSTR_TABLE_AF:
1447         case INSTR_SELECTOR:
1448         case INSTR_LEARNER:
1449         case INSTR_LEARNER_AF:
1450         case INSTR_EXTERN_OBJ:
1451         case INSTR_EXTERN_FUNC:
1452                 return 1;
1453         default:
1454                 return 0;
1455         }
1456 }
1457
1458 static struct field *
1459 action_field_parse(struct action *action, const char *name);
1460
1461 static struct field *
1462 struct_field_parse(struct rte_swx_pipeline *p,
1463                    struct action *action,
1464                    const char *name,
1465                    uint32_t *struct_id)
1466 {
1467         struct field *f;
1468
1469         switch (name[0]) {
1470         case 'h':
1471         {
1472                 struct header *header;
1473
1474                 f = header_field_parse(p, name, &header);
1475                 if (!f)
1476                         return NULL;
1477
1478                 *struct_id = header->struct_id;
1479                 return f;
1480         }
1481
1482         case 'm':
1483         {
1484                 f = metadata_field_parse(p, name);
1485                 if (!f)
1486                         return NULL;
1487
1488                 *struct_id = p->metadata_struct_id;
1489                 return f;
1490         }
1491
1492         case 't':
1493         {
1494                 if (!action)
1495                         return NULL;
1496
1497                 f = action_field_parse(action, name);
1498                 if (!f)
1499                         return NULL;
1500
1501                 *struct_id = 0;
1502                 return f;
1503         }
1504
1505         case 'e':
1506         {
1507                 struct extern_obj *obj;
1508
1509                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1510                 if (!f)
1511                         return NULL;
1512
1513                 *struct_id = obj->struct_id;
1514                 return f;
1515         }
1516
1517         case 'f':
1518         {
1519                 struct extern_func *func;
1520
1521                 f = extern_func_mailbox_field_parse(p, name, &func);
1522                 if (!f)
1523                         return NULL;
1524
1525                 *struct_id = func->struct_id;
1526                 return f;
1527         }
1528
1529         default:
1530                 return NULL;
1531         }
1532 }
1533
1534 /*
1535  * rx.
1536  */
1537 static int
1538 instr_rx_translate(struct rte_swx_pipeline *p,
1539                    struct action *action,
1540                    char **tokens,
1541                    int n_tokens,
1542                    struct instruction *instr,
1543                    struct instruction_data *data __rte_unused)
1544 {
1545         struct field *f;
1546
1547         CHECK(!action, EINVAL);
1548         CHECK(n_tokens == 2, EINVAL);
1549
1550         f = metadata_field_parse(p, tokens[1]);
1551         CHECK(f, EINVAL);
1552
1553         instr->type = INSTR_RX;
1554         instr->io.io.offset = f->offset / 8;
1555         instr->io.io.n_bits = f->n_bits;
1556         return 0;
1557 }
1558
1559 /*
1560  * tx.
1561  */
1562 static int
1563 instr_tx_translate(struct rte_swx_pipeline *p,
1564                    struct action *action __rte_unused,
1565                    char **tokens,
1566                    int n_tokens,
1567                    struct instruction *instr,
1568                    struct instruction_data *data __rte_unused)
1569 {
1570         char *port = tokens[1];
1571         struct field *f;
1572         uint32_t port_val;
1573
1574         CHECK(n_tokens == 2, EINVAL);
1575
1576         f = metadata_field_parse(p, port);
1577         if (f) {
1578                 instr->type = INSTR_TX;
1579                 instr->io.io.offset = f->offset / 8;
1580                 instr->io.io.n_bits = f->n_bits;
1581                 return 0;
1582         }
1583
1584         /* TX_I. */
1585         port_val = strtoul(port, &port, 0);
1586         CHECK(!port[0], EINVAL);
1587
1588         instr->type = INSTR_TX_I;
1589         instr->io.io.val = port_val;
1590         return 0;
1591 }
1592
1593 static int
1594 instr_drop_translate(struct rte_swx_pipeline *p,
1595                      struct action *action __rte_unused,
1596                      char **tokens __rte_unused,
1597                      int n_tokens,
1598                      struct instruction *instr,
1599                      struct instruction_data *data __rte_unused)
1600 {
1601         CHECK(n_tokens == 1, EINVAL);
1602
1603         /* TX_I. */
1604         instr->type = INSTR_TX_I;
1605         instr->io.io.val = p->n_ports_out - 1;
1606         return 0;
1607 }
1608
1609 static inline void
1610 instr_tx_exec(struct rte_swx_pipeline *p)
1611 {
1612         struct thread *t = &p->threads[p->thread_id];
1613         struct instruction *ip = t->ip;
1614
1615         __instr_tx_exec(p, t, ip);
1616
1617         /* Thread. */
1618         thread_ip_reset(p, t);
1619         instr_rx_exec(p);
1620 }
1621
1622 static inline void
1623 instr_tx_i_exec(struct rte_swx_pipeline *p)
1624 {
1625         struct thread *t = &p->threads[p->thread_id];
1626         struct instruction *ip = t->ip;
1627
1628         __instr_tx_i_exec(p, t, ip);
1629
1630         /* Thread. */
1631         thread_ip_reset(p, t);
1632         instr_rx_exec(p);
1633 }
1634
1635 /*
1636  * extract.
1637  */
1638 static int
1639 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1640                             struct action *action,
1641                             char **tokens,
1642                             int n_tokens,
1643                             struct instruction *instr,
1644                             struct instruction_data *data __rte_unused)
1645 {
1646         struct header *h;
1647
1648         CHECK(!action, EINVAL);
1649         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1650
1651         h = header_parse(p, tokens[1]);
1652         CHECK(h, EINVAL);
1653
1654         if (n_tokens == 2) {
1655                 CHECK(!h->st->var_size, EINVAL);
1656
1657                 instr->type = INSTR_HDR_EXTRACT;
1658                 instr->io.hdr.header_id[0] = h->id;
1659                 instr->io.hdr.struct_id[0] = h->struct_id;
1660                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1661         } else {
1662                 struct field *mf;
1663
1664                 CHECK(h->st->var_size, EINVAL);
1665
1666                 mf = metadata_field_parse(p, tokens[2]);
1667                 CHECK(mf, EINVAL);
1668                 CHECK(!mf->var_size, EINVAL);
1669
1670                 instr->type = INSTR_HDR_EXTRACT_M;
1671                 instr->io.io.offset = mf->offset / 8;
1672                 instr->io.io.n_bits = mf->n_bits;
1673                 instr->io.hdr.header_id[0] = h->id;
1674                 instr->io.hdr.struct_id[0] = h->struct_id;
1675                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1676         }
1677
1678         return 0;
1679 }
1680
1681 static int
1682 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1683                               struct action *action,
1684                               char **tokens,
1685                               int n_tokens,
1686                               struct instruction *instr,
1687                               struct instruction_data *data __rte_unused)
1688 {
1689         struct header *h;
1690
1691         CHECK(!action, EINVAL);
1692         CHECK(n_tokens == 2, EINVAL);
1693
1694         h = header_parse(p, tokens[1]);
1695         CHECK(h, EINVAL);
1696         CHECK(!h->st->var_size, EINVAL);
1697
1698         instr->type = INSTR_HDR_LOOKAHEAD;
1699         instr->io.hdr.header_id[0] = h->id;
1700         instr->io.hdr.struct_id[0] = h->struct_id;
1701         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1702
1703         return 0;
1704 }
1705
1706 static inline void
1707 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1708 {
1709         struct thread *t = &p->threads[p->thread_id];
1710         struct instruction *ip = t->ip;
1711
1712         __instr_hdr_extract_exec(p, t, ip);
1713
1714         /* Thread. */
1715         thread_ip_inc(p);
1716 }
1717
1718 static inline void
1719 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1720 {
1721         struct thread *t = &p->threads[p->thread_id];
1722         struct instruction *ip = t->ip;
1723
1724         __instr_hdr_extract2_exec(p, t, ip);
1725
1726         /* Thread. */
1727         thread_ip_inc(p);
1728 }
1729
1730 static inline void
1731 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1732 {
1733         struct thread *t = &p->threads[p->thread_id];
1734         struct instruction *ip = t->ip;
1735
1736         __instr_hdr_extract3_exec(p, t, ip);
1737
1738         /* Thread. */
1739         thread_ip_inc(p);
1740 }
1741
1742 static inline void
1743 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1744 {
1745         struct thread *t = &p->threads[p->thread_id];
1746         struct instruction *ip = t->ip;
1747
1748         __instr_hdr_extract4_exec(p, t, ip);
1749
1750         /* Thread. */
1751         thread_ip_inc(p);
1752 }
1753
1754 static inline void
1755 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1756 {
1757         struct thread *t = &p->threads[p->thread_id];
1758         struct instruction *ip = t->ip;
1759
1760         __instr_hdr_extract5_exec(p, t, ip);
1761
1762         /* Thread. */
1763         thread_ip_inc(p);
1764 }
1765
1766 static inline void
1767 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1768 {
1769         struct thread *t = &p->threads[p->thread_id];
1770         struct instruction *ip = t->ip;
1771
1772         __instr_hdr_extract6_exec(p, t, ip);
1773
1774         /* Thread. */
1775         thread_ip_inc(p);
1776 }
1777
1778 static inline void
1779 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1780 {
1781         struct thread *t = &p->threads[p->thread_id];
1782         struct instruction *ip = t->ip;
1783
1784         __instr_hdr_extract7_exec(p, t, ip);
1785
1786         /* Thread. */
1787         thread_ip_inc(p);
1788 }
1789
1790 static inline void
1791 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1792 {
1793         struct thread *t = &p->threads[p->thread_id];
1794         struct instruction *ip = t->ip;
1795
1796         __instr_hdr_extract8_exec(p, t, ip);
1797
1798         /* Thread. */
1799         thread_ip_inc(p);
1800 }
1801
1802 static inline void
1803 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1804 {
1805         struct thread *t = &p->threads[p->thread_id];
1806         struct instruction *ip = t->ip;
1807
1808         __instr_hdr_extract_m_exec(p, t, ip);
1809
1810         /* Thread. */
1811         thread_ip_inc(p);
1812 }
1813
1814 static inline void
1815 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1816 {
1817         struct thread *t = &p->threads[p->thread_id];
1818         struct instruction *ip = t->ip;
1819
1820         __instr_hdr_lookahead_exec(p, t, ip);
1821
1822         /* Thread. */
1823         thread_ip_inc(p);
1824 }
1825
1826 /*
1827  * emit.
1828  */
1829 static int
1830 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1831                          struct action *action __rte_unused,
1832                          char **tokens,
1833                          int n_tokens,
1834                          struct instruction *instr,
1835                          struct instruction_data *data __rte_unused)
1836 {
1837         struct header *h;
1838
1839         CHECK(n_tokens == 2, EINVAL);
1840
1841         h = header_parse(p, tokens[1]);
1842         CHECK(h, EINVAL);
1843
1844         instr->type = INSTR_HDR_EMIT;
1845         instr->io.hdr.header_id[0] = h->id;
1846         instr->io.hdr.struct_id[0] = h->struct_id;
1847         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1848         return 0;
1849 }
1850
1851 static inline void
1852 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1853 {
1854         struct thread *t = &p->threads[p->thread_id];
1855         struct instruction *ip = t->ip;
1856
1857         __instr_hdr_emit_exec(p, t, ip);
1858
1859         /* Thread. */
1860         thread_ip_inc(p);
1861 }
1862
1863 static inline void
1864 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1865 {
1866         struct thread *t = &p->threads[p->thread_id];
1867         struct instruction *ip = t->ip;
1868
1869         __instr_hdr_emit_tx_exec(p, t, ip);
1870
1871         /* Thread. */
1872         thread_ip_reset(p, t);
1873         instr_rx_exec(p);
1874 }
1875
1876 static inline void
1877 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1878 {
1879         struct thread *t = &p->threads[p->thread_id];
1880         struct instruction *ip = t->ip;
1881
1882         __instr_hdr_emit2_tx_exec(p, t, ip);
1883
1884         /* Thread. */
1885         thread_ip_reset(p, t);
1886         instr_rx_exec(p);
1887 }
1888
1889 static inline void
1890 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1891 {
1892         struct thread *t = &p->threads[p->thread_id];
1893         struct instruction *ip = t->ip;
1894
1895         __instr_hdr_emit3_tx_exec(p, t, ip);
1896
1897         /* Thread. */
1898         thread_ip_reset(p, t);
1899         instr_rx_exec(p);
1900 }
1901
1902 static inline void
1903 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1904 {
1905         struct thread *t = &p->threads[p->thread_id];
1906         struct instruction *ip = t->ip;
1907
1908         __instr_hdr_emit4_tx_exec(p, t, ip);
1909
1910         /* Thread. */
1911         thread_ip_reset(p, t);
1912         instr_rx_exec(p);
1913 }
1914
1915 static inline void
1916 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1917 {
1918         struct thread *t = &p->threads[p->thread_id];
1919         struct instruction *ip = t->ip;
1920
1921         __instr_hdr_emit5_tx_exec(p, t, ip);
1922
1923         /* Thread. */
1924         thread_ip_reset(p, t);
1925         instr_rx_exec(p);
1926 }
1927
1928 static inline void
1929 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1930 {
1931         struct thread *t = &p->threads[p->thread_id];
1932         struct instruction *ip = t->ip;
1933
1934         __instr_hdr_emit6_tx_exec(p, t, ip);
1935
1936         /* Thread. */
1937         thread_ip_reset(p, t);
1938         instr_rx_exec(p);
1939 }
1940
1941 static inline void
1942 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1943 {
1944         struct thread *t = &p->threads[p->thread_id];
1945         struct instruction *ip = t->ip;
1946
1947         __instr_hdr_emit7_tx_exec(p, t, ip);
1948
1949         /* Thread. */
1950         thread_ip_reset(p, t);
1951         instr_rx_exec(p);
1952 }
1953
1954 static inline void
1955 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1956 {
1957         struct thread *t = &p->threads[p->thread_id];
1958         struct instruction *ip = t->ip;
1959
1960         __instr_hdr_emit8_tx_exec(p, t, ip);
1961
1962         /* Thread. */
1963         thread_ip_reset(p, t);
1964         instr_rx_exec(p);
1965 }
1966
1967 /*
1968  * validate.
1969  */
1970 static int
1971 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1972                              struct action *action __rte_unused,
1973                              char **tokens,
1974                              int n_tokens,
1975                              struct instruction *instr,
1976                              struct instruction_data *data __rte_unused)
1977 {
1978         struct header *h;
1979
1980         CHECK(n_tokens == 2, EINVAL);
1981
1982         h = header_parse(p, tokens[1]);
1983         CHECK(h, EINVAL);
1984
1985         instr->type = INSTR_HDR_VALIDATE;
1986         instr->valid.header_id = h->id;
1987         return 0;
1988 }
1989
1990 static inline void
1991 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1992 {
1993         struct thread *t = &p->threads[p->thread_id];
1994         struct instruction *ip = t->ip;
1995
1996         __instr_hdr_validate_exec(p, t, ip);
1997
1998         /* Thread. */
1999         thread_ip_inc(p);
2000 }
2001
2002 /*
2003  * invalidate.
2004  */
2005 static int
2006 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2007                                struct action *action __rte_unused,
2008                                char **tokens,
2009                                int n_tokens,
2010                                struct instruction *instr,
2011                                struct instruction_data *data __rte_unused)
2012 {
2013         struct header *h;
2014
2015         CHECK(n_tokens == 2, EINVAL);
2016
2017         h = header_parse(p, tokens[1]);
2018         CHECK(h, EINVAL);
2019
2020         instr->type = INSTR_HDR_INVALIDATE;
2021         instr->valid.header_id = h->id;
2022         return 0;
2023 }
2024
2025 static inline void
2026 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2027 {
2028         struct thread *t = &p->threads[p->thread_id];
2029         struct instruction *ip = t->ip;
2030
2031         __instr_hdr_invalidate_exec(p, t, ip);
2032
2033         /* Thread. */
2034         thread_ip_inc(p);
2035 }
2036
2037 /*
2038  * table.
2039  */
2040 static struct table *
2041 table_find(struct rte_swx_pipeline *p, const char *name);
2042
2043 static struct selector *
2044 selector_find(struct rte_swx_pipeline *p, const char *name);
2045
2046 static struct learner *
2047 learner_find(struct rte_swx_pipeline *p, const char *name);
2048
2049 static int
2050 instr_table_translate(struct rte_swx_pipeline *p,
2051                       struct action *action,
2052                       char **tokens,
2053                       int n_tokens,
2054                       struct instruction *instr,
2055                       struct instruction_data *data __rte_unused)
2056 {
2057         struct table *t;
2058         struct selector *s;
2059         struct learner *l;
2060
2061         CHECK(!action, EINVAL);
2062         CHECK(n_tokens == 2, EINVAL);
2063
2064         t = table_find(p, tokens[1]);
2065         if (t) {
2066                 instr->type = INSTR_TABLE;
2067                 instr->table.table_id = t->id;
2068                 return 0;
2069         }
2070
2071         s = selector_find(p, tokens[1]);
2072         if (s) {
2073                 instr->type = INSTR_SELECTOR;
2074                 instr->table.table_id = s->id;
2075                 return 0;
2076         }
2077
2078         l = learner_find(p, tokens[1]);
2079         if (l) {
2080                 instr->type = INSTR_LEARNER;
2081                 instr->table.table_id = l->id;
2082                 return 0;
2083         }
2084
2085         CHECK(0, EINVAL);
2086 }
2087
2088 static inline void
2089 instr_table_exec(struct rte_swx_pipeline *p)
2090 {
2091         struct thread *t = &p->threads[p->thread_id];
2092         struct instruction *ip = t->ip;
2093         uint32_t table_id = ip->table.table_id;
2094         struct rte_swx_table_state *ts = &t->table_state[table_id];
2095         struct table_runtime *table = &t->tables[table_id];
2096         struct table_statistics *stats = &p->table_stats[table_id];
2097         uint64_t action_id, n_pkts_hit, n_pkts_action;
2098         uint8_t *action_data;
2099         int done, hit;
2100
2101         /* Table. */
2102         done = table->func(ts->obj,
2103                            table->mailbox,
2104                            table->key,
2105                            &action_id,
2106                            &action_data,
2107                            &hit);
2108         if (!done) {
2109                 /* Thread. */
2110                 TRACE("[Thread %2u] table %u (not finalized)\n",
2111                       p->thread_id,
2112                       table_id);
2113
2114                 thread_yield(p);
2115                 return;
2116         }
2117
2118         action_id = hit ? action_id : ts->default_action_id;
2119         action_data = hit ? action_data : ts->default_action_data;
2120         n_pkts_hit = stats->n_pkts_hit[hit];
2121         n_pkts_action = stats->n_pkts_action[action_id];
2122
2123         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2124               p->thread_id,
2125               table_id,
2126               hit ? "hit" : "miss",
2127               (uint32_t)action_id);
2128
2129         t->action_id = action_id;
2130         t->structs[0] = action_data;
2131         t->hit = hit;
2132         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2133         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2134
2135         /* Thread. */
2136         thread_ip_action_call(p, t, action_id);
2137 }
2138
2139 static inline void
2140 instr_table_af_exec(struct rte_swx_pipeline *p)
2141 {
2142         struct thread *t = &p->threads[p->thread_id];
2143         struct instruction *ip = t->ip;
2144         uint32_t table_id = ip->table.table_id;
2145         struct rte_swx_table_state *ts = &t->table_state[table_id];
2146         struct table_runtime *table = &t->tables[table_id];
2147         struct table_statistics *stats = &p->table_stats[table_id];
2148         uint64_t action_id, n_pkts_hit, n_pkts_action;
2149         uint8_t *action_data;
2150         action_func_t action_func;
2151         int done, hit;
2152
2153         /* Table. */
2154         done = table->func(ts->obj,
2155                            table->mailbox,
2156                            table->key,
2157                            &action_id,
2158                            &action_data,
2159                            &hit);
2160         if (!done) {
2161                 /* Thread. */
2162                 TRACE("[Thread %2u] table %u (not finalized)\n",
2163                       p->thread_id,
2164                       table_id);
2165
2166                 thread_yield(p);
2167                 return;
2168         }
2169
2170         action_id = hit ? action_id : ts->default_action_id;
2171         action_data = hit ? action_data : ts->default_action_data;
2172         action_func = p->action_funcs[action_id];
2173         n_pkts_hit = stats->n_pkts_hit[hit];
2174         n_pkts_action = stats->n_pkts_action[action_id];
2175
2176         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2177               p->thread_id,
2178               table_id,
2179               hit ? "hit" : "miss",
2180               (uint32_t)action_id);
2181
2182         t->action_id = action_id;
2183         t->structs[0] = action_data;
2184         t->hit = hit;
2185         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2186         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2187
2188         /* Thread. */
2189         thread_ip_inc(p);
2190
2191         /* Action. */
2192         action_func(p);
2193 }
2194
2195 static inline void
2196 instr_selector_exec(struct rte_swx_pipeline *p)
2197 {
2198         struct thread *t = &p->threads[p->thread_id];
2199         struct instruction *ip = t->ip;
2200         uint32_t selector_id = ip->table.table_id;
2201         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2202         struct selector_runtime *selector = &t->selectors[selector_id];
2203         struct selector_statistics *stats = &p->selector_stats[selector_id];
2204         uint64_t n_pkts = stats->n_pkts;
2205         int done;
2206
2207         /* Table. */
2208         done = rte_swx_table_selector_select(ts->obj,
2209                            selector->mailbox,
2210                            selector->group_id_buffer,
2211                            selector->selector_buffer,
2212                            selector->member_id_buffer);
2213         if (!done) {
2214                 /* Thread. */
2215                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2216                       p->thread_id,
2217                       selector_id);
2218
2219                 thread_yield(p);
2220                 return;
2221         }
2222
2223
2224         TRACE("[Thread %2u] selector %u\n",
2225               p->thread_id,
2226               selector_id);
2227
2228         stats->n_pkts = n_pkts + 1;
2229
2230         /* Thread. */
2231         thread_ip_inc(p);
2232 }
2233
2234 static inline void
2235 instr_learner_exec(struct rte_swx_pipeline *p)
2236 {
2237         struct thread *t = &p->threads[p->thread_id];
2238         struct instruction *ip = t->ip;
2239         uint32_t learner_id = ip->table.table_id;
2240         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2241                 p->n_selectors + learner_id];
2242         struct learner_runtime *l = &t->learners[learner_id];
2243         struct learner_statistics *stats = &p->learner_stats[learner_id];
2244         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2245         uint8_t *action_data;
2246         int done, hit;
2247
2248         /* Table. */
2249         time = rte_get_tsc_cycles();
2250
2251         done = rte_swx_table_learner_lookup(ts->obj,
2252                                             l->mailbox,
2253                                             time,
2254                                             l->key,
2255                                             &action_id,
2256                                             &action_data,
2257                                             &hit);
2258         if (!done) {
2259                 /* Thread. */
2260                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2261                       p->thread_id,
2262                       learner_id);
2263
2264                 thread_yield(p);
2265                 return;
2266         }
2267
2268         action_id = hit ? action_id : ts->default_action_id;
2269         action_data = hit ? action_data : ts->default_action_data;
2270         n_pkts_hit = stats->n_pkts_hit[hit];
2271         n_pkts_action = stats->n_pkts_action[action_id];
2272
2273         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2274               p->thread_id,
2275               learner_id,
2276               hit ? "hit" : "miss",
2277               (uint32_t)action_id);
2278
2279         t->action_id = action_id;
2280         t->structs[0] = action_data;
2281         t->hit = hit;
2282         t->learner_id = learner_id;
2283         t->time = time;
2284         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2285         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2286
2287         /* Thread. */
2288         thread_ip_action_call(p, t, action_id);
2289 }
2290
2291 static inline void
2292 instr_learner_af_exec(struct rte_swx_pipeline *p)
2293 {
2294         struct thread *t = &p->threads[p->thread_id];
2295         struct instruction *ip = t->ip;
2296         uint32_t learner_id = ip->table.table_id;
2297         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2298                 p->n_selectors + learner_id];
2299         struct learner_runtime *l = &t->learners[learner_id];
2300         struct learner_statistics *stats = &p->learner_stats[learner_id];
2301         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2302         uint8_t *action_data;
2303         action_func_t action_func;
2304         int done, hit;
2305
2306         /* Table. */
2307         time = rte_get_tsc_cycles();
2308
2309         done = rte_swx_table_learner_lookup(ts->obj,
2310                                             l->mailbox,
2311                                             time,
2312                                             l->key,
2313                                             &action_id,
2314                                             &action_data,
2315                                             &hit);
2316         if (!done) {
2317                 /* Thread. */
2318                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2319                       p->thread_id,
2320                       learner_id);
2321
2322                 thread_yield(p);
2323                 return;
2324         }
2325
2326         action_id = hit ? action_id : ts->default_action_id;
2327         action_data = hit ? action_data : ts->default_action_data;
2328         action_func = p->action_funcs[action_id];
2329         n_pkts_hit = stats->n_pkts_hit[hit];
2330         n_pkts_action = stats->n_pkts_action[action_id];
2331
2332         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2333               p->thread_id,
2334               learner_id,
2335               hit ? "hit" : "miss",
2336               (uint32_t)action_id);
2337
2338         t->action_id = action_id;
2339         t->structs[0] = action_data;
2340         t->hit = hit;
2341         t->learner_id = learner_id;
2342         t->time = time;
2343         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2344         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2345
2346         /* Thread. */
2347         thread_ip_action_call(p, t, action_id);
2348
2349         /* Action */
2350         action_func(p);
2351 }
2352
2353 /*
2354  * learn.
2355  */
2356 static struct action *
2357 action_find(struct rte_swx_pipeline *p, const char *name);
2358
2359 static int
2360 action_has_nbo_args(struct action *a);
2361
2362 static int
2363 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2364
2365 static int
2366 instr_learn_translate(struct rte_swx_pipeline *p,
2367                       struct action *action,
2368                       char **tokens,
2369                       int n_tokens,
2370                       struct instruction *instr,
2371                       struct instruction_data *data __rte_unused)
2372 {
2373         struct action *a;
2374         const char *mf_name;
2375         uint32_t mf_offset = 0;
2376
2377         CHECK(action, EINVAL);
2378         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2379
2380         a = action_find(p, tokens[1]);
2381         CHECK(a, EINVAL);
2382         CHECK(!action_has_nbo_args(a), EINVAL);
2383
2384         mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2385         CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2386
2387         if (mf_name) {
2388                 struct field *mf;
2389
2390                 mf = metadata_field_parse(p, mf_name);
2391                 CHECK(mf, EINVAL);
2392
2393                 mf_offset = mf->offset / 8;
2394         }
2395
2396         instr->type = INSTR_LEARNER_LEARN;
2397         instr->learn.action_id = a->id;
2398         instr->learn.mf_offset = mf_offset;
2399
2400         return 0;
2401 }
2402
2403 static inline void
2404 instr_learn_exec(struct rte_swx_pipeline *p)
2405 {
2406         struct thread *t = &p->threads[p->thread_id];
2407         struct instruction *ip = t->ip;
2408
2409         __instr_learn_exec(p, t, ip);
2410
2411         /* Thread. */
2412         thread_ip_inc(p);
2413 }
2414
2415 /*
2416  * forget.
2417  */
2418 static int
2419 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2420                        struct action *action,
2421                        char **tokens __rte_unused,
2422                        int n_tokens,
2423                        struct instruction *instr,
2424                        struct instruction_data *data __rte_unused)
2425 {
2426         CHECK(action, EINVAL);
2427         CHECK(n_tokens == 1, EINVAL);
2428
2429         instr->type = INSTR_LEARNER_FORGET;
2430
2431         return 0;
2432 }
2433
2434 static inline void
2435 instr_forget_exec(struct rte_swx_pipeline *p)
2436 {
2437         struct thread *t = &p->threads[p->thread_id];
2438         struct instruction *ip = t->ip;
2439
2440         __instr_forget_exec(p, t, ip);
2441
2442         /* Thread. */
2443         thread_ip_inc(p);
2444 }
2445
2446 /*
2447  * extern.
2448  */
2449 static int
2450 instr_extern_translate(struct rte_swx_pipeline *p,
2451                        struct action *action __rte_unused,
2452                        char **tokens,
2453                        int n_tokens,
2454                        struct instruction *instr,
2455                        struct instruction_data *data __rte_unused)
2456 {
2457         char *token = tokens[1];
2458
2459         CHECK(n_tokens == 2, EINVAL);
2460
2461         if (token[0] == 'e') {
2462                 struct extern_obj *obj;
2463                 struct extern_type_member_func *func;
2464
2465                 func = extern_obj_member_func_parse(p, token, &obj);
2466                 CHECK(func, EINVAL);
2467
2468                 instr->type = INSTR_EXTERN_OBJ;
2469                 instr->ext_obj.ext_obj_id = obj->id;
2470                 instr->ext_obj.func_id = func->id;
2471
2472                 return 0;
2473         }
2474
2475         if (token[0] == 'f') {
2476                 struct extern_func *func;
2477
2478                 func = extern_func_parse(p, token);
2479                 CHECK(func, EINVAL);
2480
2481                 instr->type = INSTR_EXTERN_FUNC;
2482                 instr->ext_func.ext_func_id = func->id;
2483
2484                 return 0;
2485         }
2486
2487         CHECK(0, EINVAL);
2488 }
2489
2490 static inline void
2491 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2492 {
2493         struct thread *t = &p->threads[p->thread_id];
2494         struct instruction *ip = t->ip;
2495         uint32_t done;
2496
2497         /* Extern object member function execute. */
2498         done = __instr_extern_obj_exec(p, t, ip);
2499
2500         /* Thread. */
2501         thread_ip_inc_cond(t, done);
2502         thread_yield_cond(p, done ^ 1);
2503 }
2504
2505 static inline void
2506 instr_extern_func_exec(struct rte_swx_pipeline *p)
2507 {
2508         struct thread *t = &p->threads[p->thread_id];
2509         struct instruction *ip = t->ip;
2510         uint32_t done;
2511
2512         /* Extern function execute. */
2513         done = __instr_extern_func_exec(p, t, ip);
2514
2515         /* Thread. */
2516         thread_ip_inc_cond(t, done);
2517         thread_yield_cond(p, done ^ 1);
2518 }
2519
2520 /*
2521  * mov.
2522  */
2523 static int
2524 instr_mov_translate(struct rte_swx_pipeline *p,
2525                     struct action *action,
2526                     char **tokens,
2527                     int n_tokens,
2528                     struct instruction *instr,
2529                     struct instruction_data *data __rte_unused)
2530 {
2531         char *dst = tokens[1], *src = tokens[2];
2532         struct field *fdst, *fsrc;
2533         uint64_t src_val;
2534         uint32_t dst_struct_id = 0, src_struct_id = 0;
2535
2536         CHECK(n_tokens == 3, EINVAL);
2537
2538         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2539         CHECK(fdst, EINVAL);
2540         CHECK(!fdst->var_size, EINVAL);
2541
2542         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2543         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2544         if (fsrc) {
2545                 CHECK(!fsrc->var_size, EINVAL);
2546
2547                 instr->type = INSTR_MOV;
2548                 if (dst[0] != 'h' && src[0] == 'h')
2549                         instr->type = INSTR_MOV_MH;
2550                 if (dst[0] == 'h' && src[0] != 'h')
2551                         instr->type = INSTR_MOV_HM;
2552                 if (dst[0] == 'h' && src[0] == 'h')
2553                         instr->type = INSTR_MOV_HH;
2554
2555                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2556                 instr->mov.dst.n_bits = fdst->n_bits;
2557                 instr->mov.dst.offset = fdst->offset / 8;
2558                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2559                 instr->mov.src.n_bits = fsrc->n_bits;
2560                 instr->mov.src.offset = fsrc->offset / 8;
2561                 return 0;
2562         }
2563
2564         /* MOV_I. */
2565         src_val = strtoull(src, &src, 0);
2566         CHECK(!src[0], EINVAL);
2567
2568         if (dst[0] == 'h')
2569                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2570
2571         instr->type = INSTR_MOV_I;
2572         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2573         instr->mov.dst.n_bits = fdst->n_bits;
2574         instr->mov.dst.offset = fdst->offset / 8;
2575         instr->mov.src_val = src_val;
2576         return 0;
2577 }
2578
2579 static inline void
2580 instr_mov_exec(struct rte_swx_pipeline *p)
2581 {
2582         struct thread *t = &p->threads[p->thread_id];
2583         struct instruction *ip = t->ip;
2584
2585         __instr_mov_exec(p, t, ip);
2586
2587         /* Thread. */
2588         thread_ip_inc(p);
2589 }
2590
2591 static inline void
2592 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2593 {
2594         struct thread *t = &p->threads[p->thread_id];
2595         struct instruction *ip = t->ip;
2596
2597         __instr_mov_mh_exec(p, t, ip);
2598
2599         /* Thread. */
2600         thread_ip_inc(p);
2601 }
2602
2603 static inline void
2604 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2605 {
2606         struct thread *t = &p->threads[p->thread_id];
2607         struct instruction *ip = t->ip;
2608
2609         __instr_mov_hm_exec(p, t, ip);
2610
2611         /* Thread. */
2612         thread_ip_inc(p);
2613 }
2614
2615 static inline void
2616 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2617 {
2618         struct thread *t = &p->threads[p->thread_id];
2619         struct instruction *ip = t->ip;
2620
2621         __instr_mov_hh_exec(p, t, ip);
2622
2623         /* Thread. */
2624         thread_ip_inc(p);
2625 }
2626
2627 static inline void
2628 instr_mov_i_exec(struct rte_swx_pipeline *p)
2629 {
2630         struct thread *t = &p->threads[p->thread_id];
2631         struct instruction *ip = t->ip;
2632
2633         __instr_mov_i_exec(p, t, ip);
2634
2635         /* Thread. */
2636         thread_ip_inc(p);
2637 }
2638
2639 /*
2640  * dma.
2641  */
2642 static inline void
2643 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2644 {
2645         struct thread *t = &p->threads[p->thread_id];
2646         struct instruction *ip = t->ip;
2647
2648         __instr_dma_ht_exec(p, t, ip);
2649
2650         /* Thread. */
2651         thread_ip_inc(p);
2652 }
2653
2654 static inline void
2655 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2656 {
2657         struct thread *t = &p->threads[p->thread_id];
2658         struct instruction *ip = t->ip;
2659
2660         __instr_dma_ht2_exec(p, t, ip);
2661
2662         /* Thread. */
2663         thread_ip_inc(p);
2664 }
2665
2666 static inline void
2667 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2668 {
2669         struct thread *t = &p->threads[p->thread_id];
2670         struct instruction *ip = t->ip;
2671
2672         __instr_dma_ht3_exec(p, t, ip);
2673
2674         /* Thread. */
2675         thread_ip_inc(p);
2676 }
2677
2678 static inline void
2679 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2680 {
2681         struct thread *t = &p->threads[p->thread_id];
2682         struct instruction *ip = t->ip;
2683
2684         __instr_dma_ht4_exec(p, t, ip);
2685
2686         /* Thread. */
2687         thread_ip_inc(p);
2688 }
2689
2690 static inline void
2691 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2692 {
2693         struct thread *t = &p->threads[p->thread_id];
2694         struct instruction *ip = t->ip;
2695
2696         __instr_dma_ht5_exec(p, t, ip);
2697
2698         /* Thread. */
2699         thread_ip_inc(p);
2700 }
2701
2702 static inline void
2703 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2704 {
2705         struct thread *t = &p->threads[p->thread_id];
2706         struct instruction *ip = t->ip;
2707
2708         __instr_dma_ht6_exec(p, t, ip);
2709
2710         /* Thread. */
2711         thread_ip_inc(p);
2712 }
2713
2714 static inline void
2715 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2716 {
2717         struct thread *t = &p->threads[p->thread_id];
2718         struct instruction *ip = t->ip;
2719
2720         __instr_dma_ht7_exec(p, t, ip);
2721
2722         /* Thread. */
2723         thread_ip_inc(p);
2724 }
2725
2726 static inline void
2727 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2728 {
2729         struct thread *t = &p->threads[p->thread_id];
2730         struct instruction *ip = t->ip;
2731
2732         __instr_dma_ht8_exec(p, t, ip);
2733
2734         /* Thread. */
2735         thread_ip_inc(p);
2736 }
2737
2738 /*
2739  * alu.
2740  */
2741 static int
2742 instr_alu_add_translate(struct rte_swx_pipeline *p,
2743                         struct action *action,
2744                         char **tokens,
2745                         int n_tokens,
2746                         struct instruction *instr,
2747                         struct instruction_data *data __rte_unused)
2748 {
2749         char *dst = tokens[1], *src = tokens[2];
2750         struct field *fdst, *fsrc;
2751         uint64_t src_val;
2752         uint32_t dst_struct_id = 0, src_struct_id = 0;
2753
2754         CHECK(n_tokens == 3, EINVAL);
2755
2756         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2757         CHECK(fdst, EINVAL);
2758         CHECK(!fdst->var_size, EINVAL);
2759
2760         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2761         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2762         if (fsrc) {
2763                 CHECK(!fsrc->var_size, EINVAL);
2764
2765                 instr->type = INSTR_ALU_ADD;
2766                 if (dst[0] == 'h' && src[0] != 'h')
2767                         instr->type = INSTR_ALU_ADD_HM;
2768                 if (dst[0] != 'h' && src[0] == 'h')
2769                         instr->type = INSTR_ALU_ADD_MH;
2770                 if (dst[0] == 'h' && src[0] == 'h')
2771                         instr->type = INSTR_ALU_ADD_HH;
2772
2773                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2774                 instr->alu.dst.n_bits = fdst->n_bits;
2775                 instr->alu.dst.offset = fdst->offset / 8;
2776                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2777                 instr->alu.src.n_bits = fsrc->n_bits;
2778                 instr->alu.src.offset = fsrc->offset / 8;
2779                 return 0;
2780         }
2781
2782         /* ADD_MI, ADD_HI. */
2783         src_val = strtoull(src, &src, 0);
2784         CHECK(!src[0], EINVAL);
2785
2786         instr->type = INSTR_ALU_ADD_MI;
2787         if (dst[0] == 'h')
2788                 instr->type = INSTR_ALU_ADD_HI;
2789
2790         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2791         instr->alu.dst.n_bits = fdst->n_bits;
2792         instr->alu.dst.offset = fdst->offset / 8;
2793         instr->alu.src_val = src_val;
2794         return 0;
2795 }
2796
2797 static int
2798 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2799                         struct action *action,
2800                         char **tokens,
2801                         int n_tokens,
2802                         struct instruction *instr,
2803                         struct instruction_data *data __rte_unused)
2804 {
2805         char *dst = tokens[1], *src = tokens[2];
2806         struct field *fdst, *fsrc;
2807         uint64_t src_val;
2808         uint32_t dst_struct_id = 0, src_struct_id = 0;
2809
2810         CHECK(n_tokens == 3, EINVAL);
2811
2812         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2813         CHECK(fdst, EINVAL);
2814         CHECK(!fdst->var_size, EINVAL);
2815
2816         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2817         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2818         if (fsrc) {
2819                 CHECK(!fsrc->var_size, EINVAL);
2820
2821                 instr->type = INSTR_ALU_SUB;
2822                 if (dst[0] == 'h' && src[0] != 'h')
2823                         instr->type = INSTR_ALU_SUB_HM;
2824                 if (dst[0] != 'h' && src[0] == 'h')
2825                         instr->type = INSTR_ALU_SUB_MH;
2826                 if (dst[0] == 'h' && src[0] == 'h')
2827                         instr->type = INSTR_ALU_SUB_HH;
2828
2829                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2830                 instr->alu.dst.n_bits = fdst->n_bits;
2831                 instr->alu.dst.offset = fdst->offset / 8;
2832                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2833                 instr->alu.src.n_bits = fsrc->n_bits;
2834                 instr->alu.src.offset = fsrc->offset / 8;
2835                 return 0;
2836         }
2837
2838         /* SUB_MI, SUB_HI. */
2839         src_val = strtoull(src, &src, 0);
2840         CHECK(!src[0], EINVAL);
2841
2842         instr->type = INSTR_ALU_SUB_MI;
2843         if (dst[0] == 'h')
2844                 instr->type = INSTR_ALU_SUB_HI;
2845
2846         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2847         instr->alu.dst.n_bits = fdst->n_bits;
2848         instr->alu.dst.offset = fdst->offset / 8;
2849         instr->alu.src_val = src_val;
2850         return 0;
2851 }
2852
2853 static int
2854 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2855                           struct action *action __rte_unused,
2856                           char **tokens,
2857                           int n_tokens,
2858                           struct instruction *instr,
2859                           struct instruction_data *data __rte_unused)
2860 {
2861         char *dst = tokens[1], *src = tokens[2];
2862         struct header *hdst, *hsrc;
2863         struct field *fdst, *fsrc;
2864
2865         CHECK(n_tokens == 3, EINVAL);
2866
2867         fdst = header_field_parse(p, dst, &hdst);
2868         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2869         CHECK(!fdst->var_size, EINVAL);
2870
2871         /* CKADD_FIELD. */
2872         fsrc = header_field_parse(p, src, &hsrc);
2873         if (fsrc) {
2874                 CHECK(!fsrc->var_size, EINVAL);
2875
2876                 instr->type = INSTR_ALU_CKADD_FIELD;
2877                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2878                 instr->alu.dst.n_bits = fdst->n_bits;
2879                 instr->alu.dst.offset = fdst->offset / 8;
2880                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2881                 instr->alu.src.n_bits = fsrc->n_bits;
2882                 instr->alu.src.offset = fsrc->offset / 8;
2883                 return 0;
2884         }
2885
2886         /* CKADD_STRUCT, CKADD_STRUCT20. */
2887         hsrc = header_parse(p, src);
2888         CHECK(hsrc, EINVAL);
2889         CHECK(!hsrc->st->var_size, EINVAL);
2890
2891         instr->type = INSTR_ALU_CKADD_STRUCT;
2892         if ((hsrc->st->n_bits / 8) == 20)
2893                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2894
2895         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2896         instr->alu.dst.n_bits = fdst->n_bits;
2897         instr->alu.dst.offset = fdst->offset / 8;
2898         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2899         instr->alu.src.n_bits = hsrc->st->n_bits;
2900         instr->alu.src.offset = 0; /* Unused. */
2901         return 0;
2902 }
2903
2904 static int
2905 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2906                           struct action *action __rte_unused,
2907                           char **tokens,
2908                           int n_tokens,
2909                           struct instruction *instr,
2910                           struct instruction_data *data __rte_unused)
2911 {
2912         char *dst = tokens[1], *src = tokens[2];
2913         struct header *hdst, *hsrc;
2914         struct field *fdst, *fsrc;
2915
2916         CHECK(n_tokens == 3, EINVAL);
2917
2918         fdst = header_field_parse(p, dst, &hdst);
2919         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2920         CHECK(!fdst->var_size, EINVAL);
2921
2922         fsrc = header_field_parse(p, src, &hsrc);
2923         CHECK(fsrc, EINVAL);
2924         CHECK(!fsrc->var_size, EINVAL);
2925
2926         instr->type = INSTR_ALU_CKSUB_FIELD;
2927         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2928         instr->alu.dst.n_bits = fdst->n_bits;
2929         instr->alu.dst.offset = fdst->offset / 8;
2930         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2931         instr->alu.src.n_bits = fsrc->n_bits;
2932         instr->alu.src.offset = fsrc->offset / 8;
2933         return 0;
2934 }
2935
2936 static int
2937 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2938                         struct action *action,
2939                         char **tokens,
2940                         int n_tokens,
2941                         struct instruction *instr,
2942                         struct instruction_data *data __rte_unused)
2943 {
2944         char *dst = tokens[1], *src = tokens[2];
2945         struct field *fdst, *fsrc;
2946         uint64_t src_val;
2947         uint32_t dst_struct_id = 0, src_struct_id = 0;
2948
2949         CHECK(n_tokens == 3, EINVAL);
2950
2951         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2952         CHECK(fdst, EINVAL);
2953         CHECK(!fdst->var_size, EINVAL);
2954
2955         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2956         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2957         if (fsrc) {
2958                 CHECK(!fsrc->var_size, EINVAL);
2959
2960                 instr->type = INSTR_ALU_SHL;
2961                 if (dst[0] == 'h' && src[0] != 'h')
2962                         instr->type = INSTR_ALU_SHL_HM;
2963                 if (dst[0] != 'h' && src[0] == 'h')
2964                         instr->type = INSTR_ALU_SHL_MH;
2965                 if (dst[0] == 'h' && src[0] == 'h')
2966                         instr->type = INSTR_ALU_SHL_HH;
2967
2968                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2969                 instr->alu.dst.n_bits = fdst->n_bits;
2970                 instr->alu.dst.offset = fdst->offset / 8;
2971                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2972                 instr->alu.src.n_bits = fsrc->n_bits;
2973                 instr->alu.src.offset = fsrc->offset / 8;
2974                 return 0;
2975         }
2976
2977         /* SHL_MI, SHL_HI. */
2978         src_val = strtoull(src, &src, 0);
2979         CHECK(!src[0], EINVAL);
2980
2981         instr->type = INSTR_ALU_SHL_MI;
2982         if (dst[0] == 'h')
2983                 instr->type = INSTR_ALU_SHL_HI;
2984
2985         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2986         instr->alu.dst.n_bits = fdst->n_bits;
2987         instr->alu.dst.offset = fdst->offset / 8;
2988         instr->alu.src_val = src_val;
2989         return 0;
2990 }
2991
2992 static int
2993 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2994                         struct action *action,
2995                         char **tokens,
2996                         int n_tokens,
2997                         struct instruction *instr,
2998                         struct instruction_data *data __rte_unused)
2999 {
3000         char *dst = tokens[1], *src = tokens[2];
3001         struct field *fdst, *fsrc;
3002         uint64_t src_val;
3003         uint32_t dst_struct_id = 0, src_struct_id = 0;
3004
3005         CHECK(n_tokens == 3, EINVAL);
3006
3007         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3008         CHECK(fdst, EINVAL);
3009         CHECK(!fdst->var_size, EINVAL);
3010
3011         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3012         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3013         if (fsrc) {
3014                 CHECK(!fsrc->var_size, EINVAL);
3015
3016                 instr->type = INSTR_ALU_SHR;
3017                 if (dst[0] == 'h' && src[0] != 'h')
3018                         instr->type = INSTR_ALU_SHR_HM;
3019                 if (dst[0] != 'h' && src[0] == 'h')
3020                         instr->type = INSTR_ALU_SHR_MH;
3021                 if (dst[0] == 'h' && src[0] == 'h')
3022                         instr->type = INSTR_ALU_SHR_HH;
3023
3024                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3025                 instr->alu.dst.n_bits = fdst->n_bits;
3026                 instr->alu.dst.offset = fdst->offset / 8;
3027                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3028                 instr->alu.src.n_bits = fsrc->n_bits;
3029                 instr->alu.src.offset = fsrc->offset / 8;
3030                 return 0;
3031         }
3032
3033         /* SHR_MI, SHR_HI. */
3034         src_val = strtoull(src, &src, 0);
3035         CHECK(!src[0], EINVAL);
3036
3037         instr->type = INSTR_ALU_SHR_MI;
3038         if (dst[0] == 'h')
3039                 instr->type = INSTR_ALU_SHR_HI;
3040
3041         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3042         instr->alu.dst.n_bits = fdst->n_bits;
3043         instr->alu.dst.offset = fdst->offset / 8;
3044         instr->alu.src_val = src_val;
3045         return 0;
3046 }
3047
3048 static int
3049 instr_alu_and_translate(struct rte_swx_pipeline *p,
3050                         struct action *action,
3051                         char **tokens,
3052                         int n_tokens,
3053                         struct instruction *instr,
3054                         struct instruction_data *data __rte_unused)
3055 {
3056         char *dst = tokens[1], *src = tokens[2];
3057         struct field *fdst, *fsrc;
3058         uint64_t src_val;
3059         uint32_t dst_struct_id = 0, src_struct_id = 0;
3060
3061         CHECK(n_tokens == 3, EINVAL);
3062
3063         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3064         CHECK(fdst, EINVAL);
3065         CHECK(!fdst->var_size, EINVAL);
3066
3067         /* AND, AND_MH, AND_HM, AND_HH. */
3068         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3069         if (fsrc) {
3070                 CHECK(!fsrc->var_size, EINVAL);
3071
3072                 instr->type = INSTR_ALU_AND;
3073                 if (dst[0] != 'h' && src[0] == 'h')
3074                         instr->type = INSTR_ALU_AND_MH;
3075                 if (dst[0] == 'h' && src[0] != 'h')
3076                         instr->type = INSTR_ALU_AND_HM;
3077                 if (dst[0] == 'h' && src[0] == 'h')
3078                         instr->type = INSTR_ALU_AND_HH;
3079
3080                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3081                 instr->alu.dst.n_bits = fdst->n_bits;
3082                 instr->alu.dst.offset = fdst->offset / 8;
3083                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3084                 instr->alu.src.n_bits = fsrc->n_bits;
3085                 instr->alu.src.offset = fsrc->offset / 8;
3086                 return 0;
3087         }
3088
3089         /* AND_I. */
3090         src_val = strtoull(src, &src, 0);
3091         CHECK(!src[0], EINVAL);
3092
3093         if (dst[0] == 'h')
3094                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3095
3096         instr->type = INSTR_ALU_AND_I;
3097         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3098         instr->alu.dst.n_bits = fdst->n_bits;
3099         instr->alu.dst.offset = fdst->offset / 8;
3100         instr->alu.src_val = src_val;
3101         return 0;
3102 }
3103
3104 static int
3105 instr_alu_or_translate(struct rte_swx_pipeline *p,
3106                        struct action *action,
3107                        char **tokens,
3108                        int n_tokens,
3109                        struct instruction *instr,
3110                        struct instruction_data *data __rte_unused)
3111 {
3112         char *dst = tokens[1], *src = tokens[2];
3113         struct field *fdst, *fsrc;
3114         uint64_t src_val;
3115         uint32_t dst_struct_id = 0, src_struct_id = 0;
3116
3117         CHECK(n_tokens == 3, EINVAL);
3118
3119         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3120         CHECK(fdst, EINVAL);
3121         CHECK(!fdst->var_size, EINVAL);
3122
3123         /* OR, OR_MH, OR_HM, OR_HH. */
3124         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3125         if (fsrc) {
3126                 CHECK(!fsrc->var_size, EINVAL);
3127
3128                 instr->type = INSTR_ALU_OR;
3129                 if (dst[0] != 'h' && src[0] == 'h')
3130                         instr->type = INSTR_ALU_OR_MH;
3131                 if (dst[0] == 'h' && src[0] != 'h')
3132                         instr->type = INSTR_ALU_OR_HM;
3133                 if (dst[0] == 'h' && src[0] == 'h')
3134                         instr->type = INSTR_ALU_OR_HH;
3135
3136                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3137                 instr->alu.dst.n_bits = fdst->n_bits;
3138                 instr->alu.dst.offset = fdst->offset / 8;
3139                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3140                 instr->alu.src.n_bits = fsrc->n_bits;
3141                 instr->alu.src.offset = fsrc->offset / 8;
3142                 return 0;
3143         }
3144
3145         /* OR_I. */
3146         src_val = strtoull(src, &src, 0);
3147         CHECK(!src[0], EINVAL);
3148
3149         if (dst[0] == 'h')
3150                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3151
3152         instr->type = INSTR_ALU_OR_I;
3153         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3154         instr->alu.dst.n_bits = fdst->n_bits;
3155         instr->alu.dst.offset = fdst->offset / 8;
3156         instr->alu.src_val = src_val;
3157         return 0;
3158 }
3159
3160 static int
3161 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3162                         struct action *action,
3163                         char **tokens,
3164                         int n_tokens,
3165                         struct instruction *instr,
3166                         struct instruction_data *data __rte_unused)
3167 {
3168         char *dst = tokens[1], *src = tokens[2];
3169         struct field *fdst, *fsrc;
3170         uint64_t src_val;
3171         uint32_t dst_struct_id = 0, src_struct_id = 0;
3172
3173         CHECK(n_tokens == 3, EINVAL);
3174
3175         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3176         CHECK(fdst, EINVAL);
3177         CHECK(!fdst->var_size, EINVAL);
3178
3179         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3180         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3181         if (fsrc) {
3182                 CHECK(!fsrc->var_size, EINVAL);
3183
3184                 instr->type = INSTR_ALU_XOR;
3185                 if (dst[0] != 'h' && src[0] == 'h')
3186                         instr->type = INSTR_ALU_XOR_MH;
3187                 if (dst[0] == 'h' && src[0] != 'h')
3188                         instr->type = INSTR_ALU_XOR_HM;
3189                 if (dst[0] == 'h' && src[0] == 'h')
3190                         instr->type = INSTR_ALU_XOR_HH;
3191
3192                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3193                 instr->alu.dst.n_bits = fdst->n_bits;
3194                 instr->alu.dst.offset = fdst->offset / 8;
3195                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3196                 instr->alu.src.n_bits = fsrc->n_bits;
3197                 instr->alu.src.offset = fsrc->offset / 8;
3198                 return 0;
3199         }
3200
3201         /* XOR_I. */
3202         src_val = strtoull(src, &src, 0);
3203         CHECK(!src[0], EINVAL);
3204
3205         if (dst[0] == 'h')
3206                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3207
3208         instr->type = INSTR_ALU_XOR_I;
3209         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3210         instr->alu.dst.n_bits = fdst->n_bits;
3211         instr->alu.dst.offset = fdst->offset / 8;
3212         instr->alu.src_val = src_val;
3213         return 0;
3214 }
3215
3216 static inline void
3217 instr_alu_add_exec(struct rte_swx_pipeline *p)
3218 {
3219         struct thread *t = &p->threads[p->thread_id];
3220         struct instruction *ip = t->ip;
3221
3222         /* Structs */
3223         __instr_alu_add_exec(p, t, ip);
3224
3225         /* Thread. */
3226         thread_ip_inc(p);
3227 }
3228
3229 static inline void
3230 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3231 {
3232         struct thread *t = &p->threads[p->thread_id];
3233         struct instruction *ip = t->ip;
3234
3235         /* Structs. */
3236         __instr_alu_add_mh_exec(p, t, ip);
3237
3238         /* Thread. */
3239         thread_ip_inc(p);
3240 }
3241
3242 static inline void
3243 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3244 {
3245         struct thread *t = &p->threads[p->thread_id];
3246         struct instruction *ip = t->ip;
3247
3248         /* Structs. */
3249         __instr_alu_add_hm_exec(p, t, ip);
3250
3251         /* Thread. */
3252         thread_ip_inc(p);
3253 }
3254
3255 static inline void
3256 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3257 {
3258         struct thread *t = &p->threads[p->thread_id];
3259         struct instruction *ip = t->ip;
3260
3261         /* Structs. */
3262         __instr_alu_add_hh_exec(p, t, ip);
3263
3264         /* Thread. */
3265         thread_ip_inc(p);
3266 }
3267
3268 static inline void
3269 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3270 {
3271         struct thread *t = &p->threads[p->thread_id];
3272         struct instruction *ip = t->ip;
3273
3274         /* Structs. */
3275         __instr_alu_add_mi_exec(p, t, ip);
3276
3277         /* Thread. */
3278         thread_ip_inc(p);
3279 }
3280
3281 static inline void
3282 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3283 {
3284         struct thread *t = &p->threads[p->thread_id];
3285         struct instruction *ip = t->ip;
3286
3287         /* Structs. */
3288         __instr_alu_add_hi_exec(p, t, ip);
3289
3290         /* Thread. */
3291         thread_ip_inc(p);
3292 }
3293
3294 static inline void
3295 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3296 {
3297         struct thread *t = &p->threads[p->thread_id];
3298         struct instruction *ip = t->ip;
3299
3300         /* Structs. */
3301         __instr_alu_sub_exec(p, t, ip);
3302
3303         /* Thread. */
3304         thread_ip_inc(p);
3305 }
3306
3307 static inline void
3308 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3309 {
3310         struct thread *t = &p->threads[p->thread_id];
3311         struct instruction *ip = t->ip;
3312
3313         /* Structs. */
3314         __instr_alu_sub_mh_exec(p, t, ip);
3315
3316         /* Thread. */
3317         thread_ip_inc(p);
3318 }
3319
3320 static inline void
3321 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3322 {
3323         struct thread *t = &p->threads[p->thread_id];
3324         struct instruction *ip = t->ip;
3325
3326         /* Structs. */
3327         __instr_alu_sub_hm_exec(p, t, ip);
3328
3329         /* Thread. */
3330         thread_ip_inc(p);
3331 }
3332
3333 static inline void
3334 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3335 {
3336         struct thread *t = &p->threads[p->thread_id];
3337         struct instruction *ip = t->ip;
3338
3339         /* Structs. */
3340         __instr_alu_sub_hh_exec(p, t, ip);
3341
3342         /* Thread. */
3343         thread_ip_inc(p);
3344 }
3345
3346 static inline void
3347 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3348 {
3349         struct thread *t = &p->threads[p->thread_id];
3350         struct instruction *ip = t->ip;
3351
3352         /* Structs. */
3353         __instr_alu_sub_mi_exec(p, t, ip);
3354
3355         /* Thread. */
3356         thread_ip_inc(p);
3357 }
3358
3359 static inline void
3360 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3361 {
3362         struct thread *t = &p->threads[p->thread_id];
3363         struct instruction *ip = t->ip;
3364
3365         /* Structs. */
3366         __instr_alu_sub_hi_exec(p, t, ip);
3367
3368         /* Thread. */
3369         thread_ip_inc(p);
3370 }
3371
3372 static inline void
3373 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3374 {
3375         struct thread *t = &p->threads[p->thread_id];
3376         struct instruction *ip = t->ip;
3377
3378         /* Structs. */
3379         __instr_alu_shl_exec(p, t, ip);
3380
3381         /* Thread. */
3382         thread_ip_inc(p);
3383 }
3384
3385 static inline void
3386 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3387 {
3388         struct thread *t = &p->threads[p->thread_id];
3389         struct instruction *ip = t->ip;
3390
3391         /* Structs. */
3392         __instr_alu_shl_mh_exec(p, t, ip);
3393
3394         /* Thread. */
3395         thread_ip_inc(p);
3396 }
3397
3398 static inline void
3399 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3400 {
3401         struct thread *t = &p->threads[p->thread_id];
3402         struct instruction *ip = t->ip;
3403
3404         /* Structs. */
3405         __instr_alu_shl_hm_exec(p, t, ip);
3406
3407         /* Thread. */
3408         thread_ip_inc(p);
3409 }
3410
3411 static inline void
3412 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3413 {
3414         struct thread *t = &p->threads[p->thread_id];
3415         struct instruction *ip = t->ip;
3416
3417         /* Structs. */
3418         __instr_alu_shl_hh_exec(p, t, ip);
3419
3420         /* Thread. */
3421         thread_ip_inc(p);
3422 }
3423
3424 static inline void
3425 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3426 {
3427         struct thread *t = &p->threads[p->thread_id];
3428         struct instruction *ip = t->ip;
3429
3430         /* Structs. */
3431         __instr_alu_shl_mi_exec(p, t, ip);
3432
3433         /* Thread. */
3434         thread_ip_inc(p);
3435 }
3436
3437 static inline void
3438 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3439 {
3440         struct thread *t = &p->threads[p->thread_id];
3441         struct instruction *ip = t->ip;
3442
3443         /* Structs. */
3444         __instr_alu_shl_hi_exec(p, t, ip);
3445
3446         /* Thread. */
3447         thread_ip_inc(p);
3448 }
3449
3450 static inline void
3451 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3452 {
3453         struct thread *t = &p->threads[p->thread_id];
3454         struct instruction *ip = t->ip;
3455
3456         /* Structs. */
3457         __instr_alu_shr_exec(p, t, ip);
3458
3459         /* Thread. */
3460         thread_ip_inc(p);
3461 }
3462
3463 static inline void
3464 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3465 {
3466         struct thread *t = &p->threads[p->thread_id];
3467         struct instruction *ip = t->ip;
3468
3469         /* Structs. */
3470         __instr_alu_shr_mh_exec(p, t, ip);
3471
3472         /* Thread. */
3473         thread_ip_inc(p);
3474 }
3475
3476 static inline void
3477 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3478 {
3479         struct thread *t = &p->threads[p->thread_id];
3480         struct instruction *ip = t->ip;
3481
3482         /* Structs. */
3483         __instr_alu_shr_hm_exec(p, t, ip);
3484
3485         /* Thread. */
3486         thread_ip_inc(p);
3487 }
3488
3489 static inline void
3490 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3491 {
3492         struct thread *t = &p->threads[p->thread_id];
3493         struct instruction *ip = t->ip;
3494
3495         /* Structs. */
3496         __instr_alu_shr_hh_exec(p, t, ip);
3497
3498         /* Thread. */
3499         thread_ip_inc(p);
3500 }
3501
3502 static inline void
3503 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3504 {
3505         struct thread *t = &p->threads[p->thread_id];
3506         struct instruction *ip = t->ip;
3507
3508         /* Structs. */
3509         __instr_alu_shr_mi_exec(p, t, ip);
3510
3511         /* Thread. */
3512         thread_ip_inc(p);
3513 }
3514
3515 static inline void
3516 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3517 {
3518         struct thread *t = &p->threads[p->thread_id];
3519         struct instruction *ip = t->ip;
3520
3521         /* Structs. */
3522         __instr_alu_shr_hi_exec(p, t, ip);
3523
3524         /* Thread. */
3525         thread_ip_inc(p);
3526 }
3527
3528 static inline void
3529 instr_alu_and_exec(struct rte_swx_pipeline *p)
3530 {
3531         struct thread *t = &p->threads[p->thread_id];
3532         struct instruction *ip = t->ip;
3533
3534         /* Structs. */
3535         __instr_alu_and_exec(p, t, ip);
3536
3537         /* Thread. */
3538         thread_ip_inc(p);
3539 }
3540
3541 static inline void
3542 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3543 {
3544         struct thread *t = &p->threads[p->thread_id];
3545         struct instruction *ip = t->ip;
3546
3547         /* Structs. */
3548         __instr_alu_and_mh_exec(p, t, ip);
3549
3550         /* Thread. */
3551         thread_ip_inc(p);
3552 }
3553
3554 static inline void
3555 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3556 {
3557         struct thread *t = &p->threads[p->thread_id];
3558         struct instruction *ip = t->ip;
3559
3560         /* Structs. */
3561         __instr_alu_and_hm_exec(p, t, ip);
3562
3563         /* Thread. */
3564         thread_ip_inc(p);
3565 }
3566
3567 static inline void
3568 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3569 {
3570         struct thread *t = &p->threads[p->thread_id];
3571         struct instruction *ip = t->ip;
3572
3573         /* Structs. */
3574         __instr_alu_and_hh_exec(p, t, ip);
3575
3576         /* Thread. */
3577         thread_ip_inc(p);
3578 }
3579
3580 static inline void
3581 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3582 {
3583         struct thread *t = &p->threads[p->thread_id];
3584         struct instruction *ip = t->ip;
3585
3586         /* Structs. */
3587         __instr_alu_and_i_exec(p, t, ip);
3588
3589         /* Thread. */
3590         thread_ip_inc(p);
3591 }
3592
3593 static inline void
3594 instr_alu_or_exec(struct rte_swx_pipeline *p)
3595 {
3596         struct thread *t = &p->threads[p->thread_id];
3597         struct instruction *ip = t->ip;
3598
3599         /* Structs. */
3600         __instr_alu_or_exec(p, t, ip);
3601
3602         /* Thread. */
3603         thread_ip_inc(p);
3604 }
3605
3606 static inline void
3607 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3608 {
3609         struct thread *t = &p->threads[p->thread_id];
3610         struct instruction *ip = t->ip;
3611
3612         /* Structs. */
3613         __instr_alu_or_mh_exec(p, t, ip);
3614
3615         /* Thread. */
3616         thread_ip_inc(p);
3617 }
3618
3619 static inline void
3620 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3621 {
3622         struct thread *t = &p->threads[p->thread_id];
3623         struct instruction *ip = t->ip;
3624
3625         /* Structs. */
3626         __instr_alu_or_hm_exec(p, t, ip);
3627
3628         /* Thread. */
3629         thread_ip_inc(p);
3630 }
3631
3632 static inline void
3633 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3634 {
3635         struct thread *t = &p->threads[p->thread_id];
3636         struct instruction *ip = t->ip;
3637
3638         /* Structs. */
3639         __instr_alu_or_hh_exec(p, t, ip);
3640
3641         /* Thread. */
3642         thread_ip_inc(p);
3643 }
3644
3645 static inline void
3646 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3647 {
3648         struct thread *t = &p->threads[p->thread_id];
3649         struct instruction *ip = t->ip;
3650
3651         /* Structs. */
3652         __instr_alu_or_i_exec(p, t, ip);
3653
3654         /* Thread. */
3655         thread_ip_inc(p);
3656 }
3657
3658 static inline void
3659 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3660 {
3661         struct thread *t = &p->threads[p->thread_id];
3662         struct instruction *ip = t->ip;
3663
3664         /* Structs. */
3665         __instr_alu_xor_exec(p, t, ip);
3666
3667         /* Thread. */
3668         thread_ip_inc(p);
3669 }
3670
3671 static inline void
3672 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3673 {
3674         struct thread *t = &p->threads[p->thread_id];
3675         struct instruction *ip = t->ip;
3676
3677         /* Structs. */
3678         __instr_alu_xor_mh_exec(p, t, ip);
3679
3680         /* Thread. */
3681         thread_ip_inc(p);
3682 }
3683
3684 static inline void
3685 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3686 {
3687         struct thread *t = &p->threads[p->thread_id];
3688         struct instruction *ip = t->ip;
3689
3690         /* Structs. */
3691         __instr_alu_xor_hm_exec(p, t, ip);
3692
3693         /* Thread. */
3694         thread_ip_inc(p);
3695 }
3696
3697 static inline void
3698 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3699 {
3700         struct thread *t = &p->threads[p->thread_id];
3701         struct instruction *ip = t->ip;
3702
3703         /* Structs. */
3704         __instr_alu_xor_hh_exec(p, t, ip);
3705
3706         /* Thread. */
3707         thread_ip_inc(p);
3708 }
3709
3710 static inline void
3711 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3712 {
3713         struct thread *t = &p->threads[p->thread_id];
3714         struct instruction *ip = t->ip;
3715
3716         /* Structs. */
3717         __instr_alu_xor_i_exec(p, t, ip);
3718
3719         /* Thread. */
3720         thread_ip_inc(p);
3721 }
3722
3723 static inline void
3724 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3725 {
3726         struct thread *t = &p->threads[p->thread_id];
3727         struct instruction *ip = t->ip;
3728
3729         /* Structs. */
3730         __instr_alu_ckadd_field_exec(p, t, ip);
3731
3732         /* Thread. */
3733         thread_ip_inc(p);
3734 }
3735
3736 static inline void
3737 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3738 {
3739         struct thread *t = &p->threads[p->thread_id];
3740         struct instruction *ip = t->ip;
3741
3742         /* Structs. */
3743         __instr_alu_cksub_field_exec(p, t, ip);
3744
3745         /* Thread. */
3746         thread_ip_inc(p);
3747 }
3748
3749 static inline void
3750 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3751 {
3752         struct thread *t = &p->threads[p->thread_id];
3753         struct instruction *ip = t->ip;
3754
3755         /* Structs. */
3756         __instr_alu_ckadd_struct20_exec(p, t, ip);
3757
3758         /* Thread. */
3759         thread_ip_inc(p);
3760 }
3761
3762 static inline void
3763 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3764 {
3765         struct thread *t = &p->threads[p->thread_id];
3766         struct instruction *ip = t->ip;
3767
3768         /* Structs. */
3769         __instr_alu_ckadd_struct_exec(p, t, ip);
3770
3771         /* Thread. */
3772         thread_ip_inc(p);
3773 }
3774
3775 /*
3776  * Register array.
3777  */
3778 static struct regarray *
3779 regarray_find(struct rte_swx_pipeline *p, const char *name);
3780
3781 static int
3782 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3783                       struct action *action,
3784                       char **tokens,
3785                       int n_tokens,
3786                       struct instruction *instr,
3787                       struct instruction_data *data __rte_unused)
3788 {
3789         char *regarray = tokens[1], *idx = tokens[2];
3790         struct regarray *r;
3791         struct field *fidx;
3792         uint32_t idx_struct_id, idx_val;
3793
3794         CHECK(n_tokens == 3, EINVAL);
3795
3796         r = regarray_find(p, regarray);
3797         CHECK(r, EINVAL);
3798
3799         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3800         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3801         if (fidx) {
3802                 CHECK(!fidx->var_size, EINVAL);
3803
3804                 instr->type = INSTR_REGPREFETCH_RM;
3805                 if (idx[0] == 'h')
3806                         instr->type = INSTR_REGPREFETCH_RH;
3807
3808                 instr->regarray.regarray_id = r->id;
3809                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3810                 instr->regarray.idx.n_bits = fidx->n_bits;
3811                 instr->regarray.idx.offset = fidx->offset / 8;
3812                 instr->regarray.dstsrc_val = 0; /* Unused. */
3813                 return 0;
3814         }
3815
3816         /* REGPREFETCH_RI. */
3817         idx_val = strtoul(idx, &idx, 0);
3818         CHECK(!idx[0], EINVAL);
3819
3820         instr->type = INSTR_REGPREFETCH_RI;
3821         instr->regarray.regarray_id = r->id;
3822         instr->regarray.idx_val = idx_val;
3823         instr->regarray.dstsrc_val = 0; /* Unused. */
3824         return 0;
3825 }
3826
3827 static int
3828 instr_regrd_translate(struct rte_swx_pipeline *p,
3829                       struct action *action,
3830                       char **tokens,
3831                       int n_tokens,
3832                       struct instruction *instr,
3833                       struct instruction_data *data __rte_unused)
3834 {
3835         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3836         struct regarray *r;
3837         struct field *fdst, *fidx;
3838         uint32_t dst_struct_id, idx_struct_id, idx_val;
3839
3840         CHECK(n_tokens == 4, EINVAL);
3841
3842         r = regarray_find(p, regarray);
3843         CHECK(r, EINVAL);
3844
3845         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3846         CHECK(fdst, EINVAL);
3847         CHECK(!fdst->var_size, EINVAL);
3848
3849         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3850         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3851         if (fidx) {
3852                 CHECK(!fidx->var_size, EINVAL);
3853
3854                 instr->type = INSTR_REGRD_MRM;
3855                 if (dst[0] == 'h' && idx[0] != 'h')
3856                         instr->type = INSTR_REGRD_HRM;
3857                 if (dst[0] != 'h' && idx[0] == 'h')
3858                         instr->type = INSTR_REGRD_MRH;
3859                 if (dst[0] == 'h' && idx[0] == 'h')
3860                         instr->type = INSTR_REGRD_HRH;
3861
3862                 instr->regarray.regarray_id = r->id;
3863                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3864                 instr->regarray.idx.n_bits = fidx->n_bits;
3865                 instr->regarray.idx.offset = fidx->offset / 8;
3866                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3867                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3868                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3869                 return 0;
3870         }
3871
3872         /* REGRD_MRI, REGRD_HRI. */
3873         idx_val = strtoul(idx, &idx, 0);
3874         CHECK(!idx[0], EINVAL);
3875
3876         instr->type = INSTR_REGRD_MRI;
3877         if (dst[0] == 'h')
3878                 instr->type = INSTR_REGRD_HRI;
3879
3880         instr->regarray.regarray_id = r->id;
3881         instr->regarray.idx_val = idx_val;
3882         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3883         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3884         instr->regarray.dstsrc.offset = fdst->offset / 8;
3885         return 0;
3886 }
3887
3888 static int
3889 instr_regwr_translate(struct rte_swx_pipeline *p,
3890                       struct action *action,
3891                       char **tokens,
3892                       int n_tokens,
3893                       struct instruction *instr,
3894                       struct instruction_data *data __rte_unused)
3895 {
3896         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3897         struct regarray *r;
3898         struct field *fidx, *fsrc;
3899         uint64_t src_val;
3900         uint32_t idx_struct_id, idx_val, src_struct_id;
3901
3902         CHECK(n_tokens == 4, EINVAL);
3903
3904         r = regarray_find(p, regarray);
3905         CHECK(r, EINVAL);
3906
3907         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3908         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3909         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3910         if (fidx && fsrc) {
3911                 CHECK(!fidx->var_size, EINVAL);
3912                 CHECK(!fsrc->var_size, EINVAL);
3913
3914                 instr->type = INSTR_REGWR_RMM;
3915                 if (idx[0] == 'h' && src[0] != 'h')
3916                         instr->type = INSTR_REGWR_RHM;
3917                 if (idx[0] != 'h' && src[0] == 'h')
3918                         instr->type = INSTR_REGWR_RMH;
3919                 if (idx[0] == 'h' && src[0] == 'h')
3920                         instr->type = INSTR_REGWR_RHH;
3921
3922                 instr->regarray.regarray_id = r->id;
3923                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3924                 instr->regarray.idx.n_bits = fidx->n_bits;
3925                 instr->regarray.idx.offset = fidx->offset / 8;
3926                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3927                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3928                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3929                 return 0;
3930         }
3931
3932         /* REGWR_RHI, REGWR_RMI. */
3933         if (fidx && !fsrc) {
3934                 CHECK(!fidx->var_size, EINVAL);
3935
3936                 src_val = strtoull(src, &src, 0);
3937                 CHECK(!src[0], EINVAL);
3938
3939                 instr->type = INSTR_REGWR_RMI;
3940                 if (idx[0] == 'h')
3941                         instr->type = INSTR_REGWR_RHI;
3942
3943                 instr->regarray.regarray_id = r->id;
3944                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3945                 instr->regarray.idx.n_bits = fidx->n_bits;
3946                 instr->regarray.idx.offset = fidx->offset / 8;
3947                 instr->regarray.dstsrc_val = src_val;
3948                 return 0;
3949         }
3950
3951         /* REGWR_RIH, REGWR_RIM. */
3952         if (!fidx && fsrc) {
3953                 idx_val = strtoul(idx, &idx, 0);
3954                 CHECK(!idx[0], EINVAL);
3955
3956                 CHECK(!fsrc->var_size, EINVAL);
3957
3958                 instr->type = INSTR_REGWR_RIM;
3959                 if (src[0] == 'h')
3960                         instr->type = INSTR_REGWR_RIH;
3961
3962                 instr->regarray.regarray_id = r->id;
3963                 instr->regarray.idx_val = idx_val;
3964                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3965                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3966                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3967                 return 0;
3968         }
3969
3970         /* REGWR_RII. */
3971         src_val = strtoull(src, &src, 0);
3972         CHECK(!src[0], EINVAL);
3973
3974         idx_val = strtoul(idx, &idx, 0);
3975         CHECK(!idx[0], EINVAL);
3976
3977         instr->type = INSTR_REGWR_RII;
3978         instr->regarray.idx_val = idx_val;
3979         instr->regarray.dstsrc_val = src_val;
3980
3981         return 0;
3982 }
3983
3984 static int
3985 instr_regadd_translate(struct rte_swx_pipeline *p,
3986                        struct action *action,
3987                        char **tokens,
3988                        int n_tokens,
3989                        struct instruction *instr,
3990                        struct instruction_data *data __rte_unused)
3991 {
3992         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3993         struct regarray *r;
3994         struct field *fidx, *fsrc;
3995         uint64_t src_val;
3996         uint32_t idx_struct_id, idx_val, src_struct_id;
3997
3998         CHECK(n_tokens == 4, EINVAL);
3999
4000         r = regarray_find(p, regarray);
4001         CHECK(r, EINVAL);
4002
4003         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4004         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4005         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4006         if (fidx && fsrc) {
4007                 CHECK(!fidx->var_size, EINVAL);
4008                 CHECK(!fsrc->var_size, EINVAL);
4009
4010                 instr->type = INSTR_REGADD_RMM;
4011                 if (idx[0] == 'h' && src[0] != 'h')
4012                         instr->type = INSTR_REGADD_RHM;
4013                 if (idx[0] != 'h' && src[0] == 'h')
4014                         instr->type = INSTR_REGADD_RMH;
4015                 if (idx[0] == 'h' && src[0] == 'h')
4016                         instr->type = INSTR_REGADD_RHH;
4017
4018                 instr->regarray.regarray_id = r->id;
4019                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4020                 instr->regarray.idx.n_bits = fidx->n_bits;
4021                 instr->regarray.idx.offset = fidx->offset / 8;
4022                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4023                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4024                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4025                 return 0;
4026         }
4027
4028         /* REGADD_RHI, REGADD_RMI. */
4029         if (fidx && !fsrc) {
4030                 CHECK(!fidx->var_size, EINVAL);
4031
4032                 src_val = strtoull(src, &src, 0);
4033                 CHECK(!src[0], EINVAL);
4034
4035                 instr->type = INSTR_REGADD_RMI;
4036                 if (idx[0] == 'h')
4037                         instr->type = INSTR_REGADD_RHI;
4038
4039                 instr->regarray.regarray_id = r->id;
4040                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4041                 instr->regarray.idx.n_bits = fidx->n_bits;
4042                 instr->regarray.idx.offset = fidx->offset / 8;
4043                 instr->regarray.dstsrc_val = src_val;
4044                 return 0;
4045         }
4046
4047         /* REGADD_RIH, REGADD_RIM. */
4048         if (!fidx && fsrc) {
4049                 idx_val = strtoul(idx, &idx, 0);
4050                 CHECK(!idx[0], EINVAL);
4051
4052                 CHECK(!fsrc->var_size, EINVAL);
4053
4054                 instr->type = INSTR_REGADD_RIM;
4055                 if (src[0] == 'h')
4056                         instr->type = INSTR_REGADD_RIH;
4057
4058                 instr->regarray.regarray_id = r->id;
4059                 instr->regarray.idx_val = idx_val;
4060                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4061                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4062                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4063                 return 0;
4064         }
4065
4066         /* REGADD_RII. */
4067         src_val = strtoull(src, &src, 0);
4068         CHECK(!src[0], EINVAL);
4069
4070         idx_val = strtoul(idx, &idx, 0);
4071         CHECK(!idx[0], EINVAL);
4072
4073         instr->type = INSTR_REGADD_RII;
4074         instr->regarray.idx_val = idx_val;
4075         instr->regarray.dstsrc_val = src_val;
4076         return 0;
4077 }
4078
4079 static inline void
4080 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4081 {
4082         struct thread *t = &p->threads[p->thread_id];
4083         struct instruction *ip = t->ip;
4084
4085         /* Structs. */
4086         __instr_regprefetch_rh_exec(p, t, ip);
4087
4088         /* Thread. */
4089         thread_ip_inc(p);
4090 }
4091
4092 static inline void
4093 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4094 {
4095         struct thread *t = &p->threads[p->thread_id];
4096         struct instruction *ip = t->ip;
4097
4098         /* Structs. */
4099         __instr_regprefetch_rm_exec(p, t, ip);
4100
4101         /* Thread. */
4102         thread_ip_inc(p);
4103 }
4104
4105 static inline void
4106 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4107 {
4108         struct thread *t = &p->threads[p->thread_id];
4109         struct instruction *ip = t->ip;
4110
4111         /* Structs. */
4112         __instr_regprefetch_ri_exec(p, t, ip);
4113
4114         /* Thread. */
4115         thread_ip_inc(p);
4116 }
4117
4118 static inline void
4119 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4120 {
4121         struct thread *t = &p->threads[p->thread_id];
4122         struct instruction *ip = t->ip;
4123
4124         /* Structs. */
4125         __instr_regrd_hrh_exec(p, t, ip);
4126
4127         /* Thread. */
4128         thread_ip_inc(p);
4129 }
4130
4131 static inline void
4132 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4133 {
4134         struct thread *t = &p->threads[p->thread_id];
4135         struct instruction *ip = t->ip;
4136
4137         /* Structs. */
4138         __instr_regrd_hrm_exec(p, t, ip);
4139
4140         /* Thread. */
4141         thread_ip_inc(p);
4142 }
4143
4144 static inline void
4145 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4146 {
4147         struct thread *t = &p->threads[p->thread_id];
4148         struct instruction *ip = t->ip;
4149
4150         /* Structs. */
4151         __instr_regrd_mrh_exec(p, t, ip);
4152
4153         /* Thread. */
4154         thread_ip_inc(p);
4155 }
4156
4157 static inline void
4158 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4159 {
4160         struct thread *t = &p->threads[p->thread_id];
4161         struct instruction *ip = t->ip;
4162
4163         /* Structs. */
4164         __instr_regrd_mrm_exec(p, t, ip);
4165
4166         /* Thread. */
4167         thread_ip_inc(p);
4168 }
4169
4170 static inline void
4171 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4172 {
4173         struct thread *t = &p->threads[p->thread_id];
4174         struct instruction *ip = t->ip;
4175
4176         /* Structs. */
4177         __instr_regrd_hri_exec(p, t, ip);
4178
4179         /* Thread. */
4180         thread_ip_inc(p);
4181 }
4182
4183 static inline void
4184 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4185 {
4186         struct thread *t = &p->threads[p->thread_id];
4187         struct instruction *ip = t->ip;
4188
4189         /* Structs. */
4190         __instr_regrd_mri_exec(p, t, ip);
4191
4192         /* Thread. */
4193         thread_ip_inc(p);
4194 }
4195
4196 static inline void
4197 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4198 {
4199         struct thread *t = &p->threads[p->thread_id];
4200         struct instruction *ip = t->ip;
4201
4202         /* Structs. */
4203         __instr_regwr_rhh_exec(p, t, ip);
4204
4205         /* Thread. */
4206         thread_ip_inc(p);
4207 }
4208
4209 static inline void
4210 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4211 {
4212         struct thread *t = &p->threads[p->thread_id];
4213         struct instruction *ip = t->ip;
4214
4215         /* Structs. */
4216         __instr_regwr_rhm_exec(p, t, ip);
4217
4218         /* Thread. */
4219         thread_ip_inc(p);
4220 }
4221
4222 static inline void
4223 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4224 {
4225         struct thread *t = &p->threads[p->thread_id];
4226         struct instruction *ip = t->ip;
4227
4228         /* Structs. */
4229         __instr_regwr_rmh_exec(p, t, ip);
4230
4231         /* Thread. */
4232         thread_ip_inc(p);
4233 }
4234
4235 static inline void
4236 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4237 {
4238         struct thread *t = &p->threads[p->thread_id];
4239         struct instruction *ip = t->ip;
4240
4241         /* Structs. */
4242         __instr_regwr_rmm_exec(p, t, ip);
4243
4244         /* Thread. */
4245         thread_ip_inc(p);
4246 }
4247
4248 static inline void
4249 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4250 {
4251         struct thread *t = &p->threads[p->thread_id];
4252         struct instruction *ip = t->ip;
4253
4254         /* Structs. */
4255         __instr_regwr_rhi_exec(p, t, ip);
4256
4257         /* Thread. */
4258         thread_ip_inc(p);
4259 }
4260
4261 static inline void
4262 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4263 {
4264         struct thread *t = &p->threads[p->thread_id];
4265         struct instruction *ip = t->ip;
4266
4267         /* Structs. */
4268         __instr_regwr_rmi_exec(p, t, ip);
4269
4270         /* Thread. */
4271         thread_ip_inc(p);
4272 }
4273
4274 static inline void
4275 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4276 {
4277         struct thread *t = &p->threads[p->thread_id];
4278         struct instruction *ip = t->ip;
4279
4280         /* Structs. */
4281         __instr_regwr_rih_exec(p, t, ip);
4282
4283         /* Thread. */
4284         thread_ip_inc(p);
4285 }
4286
4287 static inline void
4288 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4289 {
4290         struct thread *t = &p->threads[p->thread_id];
4291         struct instruction *ip = t->ip;
4292
4293         /* Structs. */
4294         __instr_regwr_rim_exec(p, t, ip);
4295
4296         /* Thread. */
4297         thread_ip_inc(p);
4298 }
4299
4300 static inline void
4301 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4302 {
4303         struct thread *t = &p->threads[p->thread_id];
4304         struct instruction *ip = t->ip;
4305
4306         /* Structs. */
4307         __instr_regwr_rii_exec(p, t, ip);
4308
4309         /* Thread. */
4310         thread_ip_inc(p);
4311 }
4312
4313 static inline void
4314 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4315 {
4316         struct thread *t = &p->threads[p->thread_id];
4317         struct instruction *ip = t->ip;
4318
4319         /* Structs. */
4320         __instr_regadd_rhh_exec(p, t, ip);
4321
4322         /* Thread. */
4323         thread_ip_inc(p);
4324 }
4325
4326 static inline void
4327 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4328 {
4329         struct thread *t = &p->threads[p->thread_id];
4330         struct instruction *ip = t->ip;
4331
4332         /* Structs. */
4333         __instr_regadd_rhm_exec(p, t, ip);
4334
4335         /* Thread. */
4336         thread_ip_inc(p);
4337 }
4338
4339 static inline void
4340 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4341 {
4342         struct thread *t = &p->threads[p->thread_id];
4343         struct instruction *ip = t->ip;
4344
4345         /* Structs. */
4346         __instr_regadd_rmh_exec(p, t, ip);
4347
4348         /* Thread. */
4349         thread_ip_inc(p);
4350 }
4351
4352 static inline void
4353 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4354 {
4355         struct thread *t = &p->threads[p->thread_id];
4356         struct instruction *ip = t->ip;
4357
4358         /* Structs. */
4359         __instr_regadd_rmm_exec(p, t, ip);
4360
4361         /* Thread. */
4362         thread_ip_inc(p);
4363 }
4364
4365 static inline void
4366 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4367 {
4368         struct thread *t = &p->threads[p->thread_id];
4369         struct instruction *ip = t->ip;
4370
4371         /* Structs. */
4372         __instr_regadd_rhi_exec(p, t, ip);
4373
4374         /* Thread. */
4375         thread_ip_inc(p);
4376 }
4377
4378 static inline void
4379 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4380 {
4381         struct thread *t = &p->threads[p->thread_id];
4382         struct instruction *ip = t->ip;
4383
4384         /* Structs. */
4385         __instr_regadd_rmi_exec(p, t, ip);
4386
4387         /* Thread. */
4388         thread_ip_inc(p);
4389 }
4390
4391 static inline void
4392 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4393 {
4394         struct thread *t = &p->threads[p->thread_id];
4395         struct instruction *ip = t->ip;
4396
4397         /* Structs. */
4398         __instr_regadd_rih_exec(p, t, ip);
4399
4400         /* Thread. */
4401         thread_ip_inc(p);
4402 }
4403
4404 static inline void
4405 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4406 {
4407         struct thread *t = &p->threads[p->thread_id];
4408         struct instruction *ip = t->ip;
4409
4410         /* Structs. */
4411         __instr_regadd_rim_exec(p, t, ip);
4412
4413         /* Thread. */
4414         thread_ip_inc(p);
4415 }
4416
4417 static inline void
4418 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4419 {
4420         struct thread *t = &p->threads[p->thread_id];
4421         struct instruction *ip = t->ip;
4422
4423         /* Structs. */
4424         __instr_regadd_rii_exec(p, t, ip);
4425
4426         /* Thread. */
4427         thread_ip_inc(p);
4428 }
4429
4430 /*
4431  * metarray.
4432  */
4433 static struct metarray *
4434 metarray_find(struct rte_swx_pipeline *p, const char *name);
4435
4436 static int
4437 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4438                             struct action *action,
4439                             char **tokens,
4440                             int n_tokens,
4441                             struct instruction *instr,
4442                             struct instruction_data *data __rte_unused)
4443 {
4444         char *metarray = tokens[1], *idx = tokens[2];
4445         struct metarray *m;
4446         struct field *fidx;
4447         uint32_t idx_struct_id, idx_val;
4448
4449         CHECK(n_tokens == 3, EINVAL);
4450
4451         m = metarray_find(p, metarray);
4452         CHECK(m, EINVAL);
4453
4454         /* METPREFETCH_H, METPREFETCH_M. */
4455         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4456         if (fidx) {
4457                 CHECK(!fidx->var_size, EINVAL);
4458
4459                 instr->type = INSTR_METPREFETCH_M;
4460                 if (idx[0] == 'h')
4461                         instr->type = INSTR_METPREFETCH_H;
4462
4463                 instr->meter.metarray_id = m->id;
4464                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4465                 instr->meter.idx.n_bits = fidx->n_bits;
4466                 instr->meter.idx.offset = fidx->offset / 8;
4467                 return 0;
4468         }
4469
4470         /* METPREFETCH_I. */
4471         idx_val = strtoul(idx, &idx, 0);
4472         CHECK(!idx[0], EINVAL);
4473
4474         instr->type = INSTR_METPREFETCH_I;
4475         instr->meter.metarray_id = m->id;
4476         instr->meter.idx_val = idx_val;
4477         return 0;
4478 }
4479
4480 static int
4481 instr_meter_translate(struct rte_swx_pipeline *p,
4482                       struct action *action,
4483                       char **tokens,
4484                       int n_tokens,
4485                       struct instruction *instr,
4486                       struct instruction_data *data __rte_unused)
4487 {
4488         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4489         char *color_in = tokens[4], *color_out = tokens[5];
4490         struct metarray *m;
4491         struct field *fidx, *flength, *fcin, *fcout;
4492         uint32_t idx_struct_id, length_struct_id;
4493         uint32_t color_in_struct_id, color_out_struct_id;
4494
4495         CHECK(n_tokens == 6, EINVAL);
4496
4497         m = metarray_find(p, metarray);
4498         CHECK(m, EINVAL);
4499
4500         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4501
4502         flength = struct_field_parse(p, action, length, &length_struct_id);
4503         CHECK(flength, EINVAL);
4504         CHECK(!flength->var_size, EINVAL);
4505
4506         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4507
4508         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4509         CHECK(fcout, EINVAL);
4510         CHECK(!fcout->var_size, EINVAL);
4511
4512         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4513         if (fidx && fcin) {
4514                 CHECK(!fidx->var_size, EINVAL);
4515                 CHECK(!fcin->var_size, EINVAL);
4516
4517                 instr->type = INSTR_METER_MMM;
4518                 if (idx[0] == 'h' && length[0] == 'h')
4519                         instr->type = INSTR_METER_HHM;
4520                 if (idx[0] == 'h' && length[0] != 'h')
4521                         instr->type = INSTR_METER_HMM;
4522                 if (idx[0] != 'h' && length[0] == 'h')
4523                         instr->type = INSTR_METER_MHM;
4524
4525                 instr->meter.metarray_id = m->id;
4526
4527                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4528                 instr->meter.idx.n_bits = fidx->n_bits;
4529                 instr->meter.idx.offset = fidx->offset / 8;
4530
4531                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4532                 instr->meter.length.n_bits = flength->n_bits;
4533                 instr->meter.length.offset = flength->offset / 8;
4534
4535                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4536                 instr->meter.color_in.n_bits = fcin->n_bits;
4537                 instr->meter.color_in.offset = fcin->offset / 8;
4538
4539                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4540                 instr->meter.color_out.n_bits = fcout->n_bits;
4541                 instr->meter.color_out.offset = fcout->offset / 8;
4542
4543                 return 0;
4544         }
4545
4546         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4547         if (fidx && !fcin) {
4548                 uint32_t color_in_val;
4549
4550                 CHECK(!fidx->var_size, EINVAL);
4551
4552                 color_in_val = strtoul(color_in, &color_in, 0);
4553                 CHECK(!color_in[0], EINVAL);
4554
4555                 instr->type = INSTR_METER_MMI;
4556                 if (idx[0] == 'h' && length[0] == 'h')
4557                         instr->type = INSTR_METER_HHI;
4558                 if (idx[0] == 'h' && length[0] != 'h')
4559                         instr->type = INSTR_METER_HMI;
4560                 if (idx[0] != 'h' && length[0] == 'h')
4561                         instr->type = INSTR_METER_MHI;
4562
4563                 instr->meter.metarray_id = m->id;
4564
4565                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4566                 instr->meter.idx.n_bits = fidx->n_bits;
4567                 instr->meter.idx.offset = fidx->offset / 8;
4568
4569                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4570                 instr->meter.length.n_bits = flength->n_bits;
4571                 instr->meter.length.offset = flength->offset / 8;
4572
4573                 instr->meter.color_in_val = color_in_val;
4574
4575                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4576                 instr->meter.color_out.n_bits = fcout->n_bits;
4577                 instr->meter.color_out.offset = fcout->offset / 8;
4578
4579                 return 0;
4580         }
4581
4582         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4583         if (!fidx && fcin) {
4584                 uint32_t idx_val;
4585
4586                 idx_val = strtoul(idx, &idx, 0);
4587                 CHECK(!idx[0], EINVAL);
4588
4589                 CHECK(!fcin->var_size, EINVAL);
4590
4591                 instr->type = INSTR_METER_IMM;
4592                 if (length[0] == 'h')
4593                         instr->type = INSTR_METER_IHM;
4594
4595                 instr->meter.metarray_id = m->id;
4596
4597                 instr->meter.idx_val = idx_val;
4598
4599                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4600                 instr->meter.length.n_bits = flength->n_bits;
4601                 instr->meter.length.offset = flength->offset / 8;
4602
4603                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4604                 instr->meter.color_in.n_bits = fcin->n_bits;
4605                 instr->meter.color_in.offset = fcin->offset / 8;
4606
4607                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4608                 instr->meter.color_out.n_bits = fcout->n_bits;
4609                 instr->meter.color_out.offset = fcout->offset / 8;
4610
4611                 return 0;
4612         }
4613
4614         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4615         if (!fidx && !fcin) {
4616                 uint32_t idx_val, color_in_val;
4617
4618                 idx_val = strtoul(idx, &idx, 0);
4619                 CHECK(!idx[0], EINVAL);
4620
4621                 color_in_val = strtoul(color_in, &color_in, 0);
4622                 CHECK(!color_in[0], EINVAL);
4623
4624                 instr->type = INSTR_METER_IMI;
4625                 if (length[0] == 'h')
4626                         instr->type = INSTR_METER_IHI;
4627
4628                 instr->meter.metarray_id = m->id;
4629
4630                 instr->meter.idx_val = idx_val;
4631
4632                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4633                 instr->meter.length.n_bits = flength->n_bits;
4634                 instr->meter.length.offset = flength->offset / 8;
4635
4636                 instr->meter.color_in_val = color_in_val;
4637
4638                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4639                 instr->meter.color_out.n_bits = fcout->n_bits;
4640                 instr->meter.color_out.offset = fcout->offset / 8;
4641
4642                 return 0;
4643         }
4644
4645         return -EINVAL;
4646 }
4647
4648 static inline void
4649 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4650 {
4651         struct thread *t = &p->threads[p->thread_id];
4652         struct instruction *ip = t->ip;
4653
4654         /* Structs. */
4655         __instr_metprefetch_h_exec(p, t, ip);
4656
4657         /* Thread. */
4658         thread_ip_inc(p);
4659 }
4660
4661 static inline void
4662 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4663 {
4664         struct thread *t = &p->threads[p->thread_id];
4665         struct instruction *ip = t->ip;
4666
4667         /* Structs. */
4668         __instr_metprefetch_m_exec(p, t, ip);
4669
4670         /* Thread. */
4671         thread_ip_inc(p);
4672 }
4673
4674 static inline void
4675 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4676 {
4677         struct thread *t = &p->threads[p->thread_id];
4678         struct instruction *ip = t->ip;
4679
4680         /* Structs. */
4681         __instr_metprefetch_i_exec(p, t, ip);
4682
4683         /* Thread. */
4684         thread_ip_inc(p);
4685 }
4686
4687 static inline void
4688 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4689 {
4690         struct thread *t = &p->threads[p->thread_id];
4691         struct instruction *ip = t->ip;
4692
4693         /* Structs. */
4694         __instr_meter_hhm_exec(p, t, ip);
4695
4696         /* Thread. */
4697         thread_ip_inc(p);
4698 }
4699
4700 static inline void
4701 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4702 {
4703         struct thread *t = &p->threads[p->thread_id];
4704         struct instruction *ip = t->ip;
4705
4706         /* Structs. */
4707         __instr_meter_hhi_exec(p, t, ip);
4708
4709         /* Thread. */
4710         thread_ip_inc(p);
4711 }
4712
4713 static inline void
4714 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4715 {
4716         struct thread *t = &p->threads[p->thread_id];
4717         struct instruction *ip = t->ip;
4718
4719         /* Structs. */
4720         __instr_meter_hmm_exec(p, t, ip);
4721
4722         /* Thread. */
4723         thread_ip_inc(p);
4724 }
4725
4726 static inline void
4727 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4728 {
4729         struct thread *t = &p->threads[p->thread_id];
4730         struct instruction *ip = t->ip;
4731
4732         /* Structs. */
4733         __instr_meter_hmi_exec(p, t, ip);
4734
4735         /* Thread. */
4736         thread_ip_inc(p);
4737 }
4738
4739 static inline void
4740 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4741 {
4742         struct thread *t = &p->threads[p->thread_id];
4743         struct instruction *ip = t->ip;
4744
4745         /* Structs. */
4746         __instr_meter_mhm_exec(p, t, ip);
4747
4748         /* Thread. */
4749         thread_ip_inc(p);
4750 }
4751
4752 static inline void
4753 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4754 {
4755         struct thread *t = &p->threads[p->thread_id];
4756         struct instruction *ip = t->ip;
4757
4758         /* Structs. */
4759         __instr_meter_mhi_exec(p, t, ip);
4760
4761         /* Thread. */
4762         thread_ip_inc(p);
4763 }
4764
4765 static inline void
4766 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4767 {
4768         struct thread *t = &p->threads[p->thread_id];
4769         struct instruction *ip = t->ip;
4770
4771         /* Structs. */
4772         __instr_meter_mmm_exec(p, t, ip);
4773
4774         /* Thread. */
4775         thread_ip_inc(p);
4776 }
4777
4778 static inline void
4779 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4780 {
4781         struct thread *t = &p->threads[p->thread_id];
4782         struct instruction *ip = t->ip;
4783
4784         /* Structs. */
4785         __instr_meter_mmi_exec(p, t, ip);
4786
4787         /* Thread. */
4788         thread_ip_inc(p);
4789 }
4790
4791 static inline void
4792 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4793 {
4794         struct thread *t = &p->threads[p->thread_id];
4795         struct instruction *ip = t->ip;
4796
4797         /* Structs. */
4798         __instr_meter_ihm_exec(p, t, ip);
4799
4800         /* Thread. */
4801         thread_ip_inc(p);
4802 }
4803
4804 static inline void
4805 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4806 {
4807         struct thread *t = &p->threads[p->thread_id];
4808         struct instruction *ip = t->ip;
4809
4810         /* Structs. */
4811         __instr_meter_ihi_exec(p, t, ip);
4812
4813         /* Thread. */
4814         thread_ip_inc(p);
4815 }
4816
4817 static inline void
4818 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4819 {
4820         struct thread *t = &p->threads[p->thread_id];
4821         struct instruction *ip = t->ip;
4822
4823         /* Structs. */
4824         __instr_meter_imm_exec(p, t, ip);
4825
4826         /* Thread. */
4827         thread_ip_inc(p);
4828 }
4829
4830 static inline void
4831 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4832 {
4833         struct thread *t = &p->threads[p->thread_id];
4834         struct instruction *ip = t->ip;
4835
4836         /* Structs. */
4837         __instr_meter_imi_exec(p, t, ip);
4838
4839         /* Thread. */
4840         thread_ip_inc(p);
4841 }
4842
4843 /*
4844  * jmp.
4845  */
4846 static int
4847 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4848                     struct action *action __rte_unused,
4849                     char **tokens,
4850                     int n_tokens,
4851                     struct instruction *instr,
4852                     struct instruction_data *data)
4853 {
4854         CHECK(n_tokens == 2, EINVAL);
4855
4856         strcpy(data->jmp_label, tokens[1]);
4857
4858         instr->type = INSTR_JMP;
4859         instr->jmp.ip = NULL; /* Resolved later. */
4860         return 0;
4861 }
4862
4863 static int
4864 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4865                           struct action *action __rte_unused,
4866                           char **tokens,
4867                           int n_tokens,
4868                           struct instruction *instr,
4869                           struct instruction_data *data)
4870 {
4871         struct header *h;
4872
4873         CHECK(n_tokens == 3, EINVAL);
4874
4875         strcpy(data->jmp_label, tokens[1]);
4876
4877         h = header_parse(p, tokens[2]);
4878         CHECK(h, EINVAL);
4879
4880         instr->type = INSTR_JMP_VALID;
4881         instr->jmp.ip = NULL; /* Resolved later. */
4882         instr->jmp.header_id = h->id;
4883         return 0;
4884 }
4885
4886 static int
4887 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4888                             struct action *action __rte_unused,
4889                             char **tokens,
4890                             int n_tokens,
4891                             struct instruction *instr,
4892                             struct instruction_data *data)
4893 {
4894         struct header *h;
4895
4896         CHECK(n_tokens == 3, EINVAL);
4897
4898         strcpy(data->jmp_label, tokens[1]);
4899
4900         h = header_parse(p, tokens[2]);
4901         CHECK(h, EINVAL);
4902
4903         instr->type = INSTR_JMP_INVALID;
4904         instr->jmp.ip = NULL; /* Resolved later. */
4905         instr->jmp.header_id = h->id;
4906         return 0;
4907 }
4908
4909 static int
4910 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4911                         struct action *action,
4912                         char **tokens,
4913                         int n_tokens,
4914                         struct instruction *instr,
4915                         struct instruction_data *data)
4916 {
4917         CHECK(!action, EINVAL);
4918         CHECK(n_tokens == 2, EINVAL);
4919
4920         strcpy(data->jmp_label, tokens[1]);
4921
4922         instr->type = INSTR_JMP_HIT;
4923         instr->jmp.ip = NULL; /* Resolved later. */
4924         return 0;
4925 }
4926
4927 static int
4928 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4929                          struct action *action,
4930                          char **tokens,
4931                          int n_tokens,
4932                          struct instruction *instr,
4933                          struct instruction_data *data)
4934 {
4935         CHECK(!action, EINVAL);
4936         CHECK(n_tokens == 2, EINVAL);
4937
4938         strcpy(data->jmp_label, tokens[1]);
4939
4940         instr->type = INSTR_JMP_MISS;
4941         instr->jmp.ip = NULL; /* Resolved later. */
4942         return 0;
4943 }
4944
4945 static int
4946 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4947                                struct action *action,
4948                                char **tokens,
4949                                int n_tokens,
4950                                struct instruction *instr,
4951                                struct instruction_data *data)
4952 {
4953         struct action *a;
4954
4955         CHECK(!action, EINVAL);
4956         CHECK(n_tokens == 3, EINVAL);
4957
4958         strcpy(data->jmp_label, tokens[1]);
4959
4960         a = action_find(p, tokens[2]);
4961         CHECK(a, EINVAL);
4962
4963         instr->type = INSTR_JMP_ACTION_HIT;
4964         instr->jmp.ip = NULL; /* Resolved later. */
4965         instr->jmp.action_id = a->id;
4966         return 0;
4967 }
4968
4969 static int
4970 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4971                                 struct action *action,
4972                                 char **tokens,
4973                                 int n_tokens,
4974                                 struct instruction *instr,
4975                                 struct instruction_data *data)
4976 {
4977         struct action *a;
4978
4979         CHECK(!action, EINVAL);
4980         CHECK(n_tokens == 3, EINVAL);
4981
4982         strcpy(data->jmp_label, tokens[1]);
4983
4984         a = action_find(p, tokens[2]);
4985         CHECK(a, EINVAL);
4986
4987         instr->type = INSTR_JMP_ACTION_MISS;
4988         instr->jmp.ip = NULL; /* Resolved later. */
4989         instr->jmp.action_id = a->id;
4990         return 0;
4991 }
4992
4993 static int
4994 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4995                        struct action *action,
4996                        char **tokens,
4997                        int n_tokens,
4998                        struct instruction *instr,
4999                        struct instruction_data *data)
5000 {
5001         char *a = tokens[2], *b = tokens[3];
5002         struct field *fa, *fb;
5003         uint64_t b_val;
5004         uint32_t a_struct_id, b_struct_id;
5005
5006         CHECK(n_tokens == 4, EINVAL);
5007
5008         strcpy(data->jmp_label, tokens[1]);
5009
5010         fa = struct_field_parse(p, action, a, &a_struct_id);
5011         CHECK(fa, EINVAL);
5012         CHECK(!fa->var_size, EINVAL);
5013
5014         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5015         fb = struct_field_parse(p, action, b, &b_struct_id);
5016         if (fb) {
5017                 CHECK(!fb->var_size, EINVAL);
5018
5019                 instr->type = INSTR_JMP_EQ;
5020                 if (a[0] != 'h' && b[0] == 'h')
5021                         instr->type = INSTR_JMP_EQ_MH;
5022                 if (a[0] == 'h' && b[0] != 'h')
5023                         instr->type = INSTR_JMP_EQ_HM;
5024                 if (a[0] == 'h' && b[0] == 'h')
5025                         instr->type = INSTR_JMP_EQ_HH;
5026                 instr->jmp.ip = NULL; /* Resolved later. */
5027
5028                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5029                 instr->jmp.a.n_bits = fa->n_bits;
5030                 instr->jmp.a.offset = fa->offset / 8;
5031                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5032                 instr->jmp.b.n_bits = fb->n_bits;
5033                 instr->jmp.b.offset = fb->offset / 8;
5034                 return 0;
5035         }
5036
5037         /* JMP_EQ_I. */
5038         b_val = strtoull(b, &b, 0);
5039         CHECK(!b[0], EINVAL);
5040
5041         if (a[0] == 'h')
5042                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5043
5044         instr->type = INSTR_JMP_EQ_I;
5045         instr->jmp.ip = NULL; /* Resolved later. */
5046         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5047         instr->jmp.a.n_bits = fa->n_bits;
5048         instr->jmp.a.offset = fa->offset / 8;
5049         instr->jmp.b_val = b_val;
5050         return 0;
5051 }
5052
5053 static int
5054 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5055                         struct action *action,
5056                         char **tokens,
5057                         int n_tokens,
5058                         struct instruction *instr,
5059                         struct instruction_data *data)
5060 {
5061         char *a = tokens[2], *b = tokens[3];
5062         struct field *fa, *fb;
5063         uint64_t b_val;
5064         uint32_t a_struct_id, b_struct_id;
5065
5066         CHECK(n_tokens == 4, EINVAL);
5067
5068         strcpy(data->jmp_label, tokens[1]);
5069
5070         fa = struct_field_parse(p, action, a, &a_struct_id);
5071         CHECK(fa, EINVAL);
5072         CHECK(!fa->var_size, EINVAL);
5073
5074         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5075         fb = struct_field_parse(p, action, b, &b_struct_id);
5076         if (fb) {
5077                 CHECK(!fb->var_size, EINVAL);
5078
5079                 instr->type = INSTR_JMP_NEQ;
5080                 if (a[0] != 'h' && b[0] == 'h')
5081                         instr->type = INSTR_JMP_NEQ_MH;
5082                 if (a[0] == 'h' && b[0] != 'h')
5083                         instr->type = INSTR_JMP_NEQ_HM;
5084                 if (a[0] == 'h' && b[0] == 'h')
5085                         instr->type = INSTR_JMP_NEQ_HH;
5086                 instr->jmp.ip = NULL; /* Resolved later. */
5087
5088                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5089                 instr->jmp.a.n_bits = fa->n_bits;
5090                 instr->jmp.a.offset = fa->offset / 8;
5091                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5092                 instr->jmp.b.n_bits = fb->n_bits;
5093                 instr->jmp.b.offset = fb->offset / 8;
5094                 return 0;
5095         }
5096
5097         /* JMP_NEQ_I. */
5098         b_val = strtoull(b, &b, 0);
5099         CHECK(!b[0], EINVAL);
5100
5101         if (a[0] == 'h')
5102                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5103
5104         instr->type = INSTR_JMP_NEQ_I;
5105         instr->jmp.ip = NULL; /* Resolved later. */
5106         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5107         instr->jmp.a.n_bits = fa->n_bits;
5108         instr->jmp.a.offset = fa->offset / 8;
5109         instr->jmp.b_val = b_val;
5110         return 0;
5111 }
5112
5113 static int
5114 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5115                        struct action *action,
5116                        char **tokens,
5117                        int n_tokens,
5118                        struct instruction *instr,
5119                        struct instruction_data *data)
5120 {
5121         char *a = tokens[2], *b = tokens[3];
5122         struct field *fa, *fb;
5123         uint64_t b_val;
5124         uint32_t a_struct_id, b_struct_id;
5125
5126         CHECK(n_tokens == 4, EINVAL);
5127
5128         strcpy(data->jmp_label, tokens[1]);
5129
5130         fa = struct_field_parse(p, action, a, &a_struct_id);
5131         CHECK(fa, EINVAL);
5132         CHECK(!fa->var_size, EINVAL);
5133
5134         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5135         fb = struct_field_parse(p, action, b, &b_struct_id);
5136         if (fb) {
5137                 CHECK(!fb->var_size, EINVAL);
5138
5139                 instr->type = INSTR_JMP_LT;
5140                 if (a[0] == 'h' && b[0] != 'h')
5141                         instr->type = INSTR_JMP_LT_HM;
5142                 if (a[0] != 'h' && b[0] == 'h')
5143                         instr->type = INSTR_JMP_LT_MH;
5144                 if (a[0] == 'h' && b[0] == 'h')
5145                         instr->type = INSTR_JMP_LT_HH;
5146                 instr->jmp.ip = NULL; /* Resolved later. */
5147
5148                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5149                 instr->jmp.a.n_bits = fa->n_bits;
5150                 instr->jmp.a.offset = fa->offset / 8;
5151                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5152                 instr->jmp.b.n_bits = fb->n_bits;
5153                 instr->jmp.b.offset = fb->offset / 8;
5154                 return 0;
5155         }
5156
5157         /* JMP_LT_MI, JMP_LT_HI. */
5158         b_val = strtoull(b, &b, 0);
5159         CHECK(!b[0], EINVAL);
5160
5161         instr->type = INSTR_JMP_LT_MI;
5162         if (a[0] == 'h')
5163                 instr->type = INSTR_JMP_LT_HI;
5164         instr->jmp.ip = NULL; /* Resolved later. */
5165
5166         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5167         instr->jmp.a.n_bits = fa->n_bits;
5168         instr->jmp.a.offset = fa->offset / 8;
5169         instr->jmp.b_val = b_val;
5170         return 0;
5171 }
5172
5173 static int
5174 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5175                        struct action *action,
5176                        char **tokens,
5177                        int n_tokens,
5178                        struct instruction *instr,
5179                        struct instruction_data *data)
5180 {
5181         char *a = tokens[2], *b = tokens[3];
5182         struct field *fa, *fb;
5183         uint64_t b_val;
5184         uint32_t a_struct_id, b_struct_id;
5185
5186         CHECK(n_tokens == 4, EINVAL);
5187
5188         strcpy(data->jmp_label, tokens[1]);
5189
5190         fa = struct_field_parse(p, action, a, &a_struct_id);
5191         CHECK(fa, EINVAL);
5192         CHECK(!fa->var_size, EINVAL);
5193
5194         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5195         fb = struct_field_parse(p, action, b, &b_struct_id);
5196         if (fb) {
5197                 CHECK(!fb->var_size, EINVAL);
5198
5199                 instr->type = INSTR_JMP_GT;
5200                 if (a[0] == 'h' && b[0] != 'h')
5201                         instr->type = INSTR_JMP_GT_HM;
5202                 if (a[0] != 'h' && b[0] == 'h')
5203                         instr->type = INSTR_JMP_GT_MH;
5204                 if (a[0] == 'h' && b[0] == 'h')
5205                         instr->type = INSTR_JMP_GT_HH;
5206                 instr->jmp.ip = NULL; /* Resolved later. */
5207
5208                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5209                 instr->jmp.a.n_bits = fa->n_bits;
5210                 instr->jmp.a.offset = fa->offset / 8;
5211                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5212                 instr->jmp.b.n_bits = fb->n_bits;
5213                 instr->jmp.b.offset = fb->offset / 8;
5214                 return 0;
5215         }
5216
5217         /* JMP_GT_MI, JMP_GT_HI. */
5218         b_val = strtoull(b, &b, 0);
5219         CHECK(!b[0], EINVAL);
5220
5221         instr->type = INSTR_JMP_GT_MI;
5222         if (a[0] == 'h')
5223                 instr->type = INSTR_JMP_GT_HI;
5224         instr->jmp.ip = NULL; /* Resolved later. */
5225
5226         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5227         instr->jmp.a.n_bits = fa->n_bits;
5228         instr->jmp.a.offset = fa->offset / 8;
5229         instr->jmp.b_val = b_val;
5230         return 0;
5231 }
5232
5233 static inline void
5234 instr_jmp_exec(struct rte_swx_pipeline *p)
5235 {
5236         struct thread *t = &p->threads[p->thread_id];
5237         struct instruction *ip = t->ip;
5238
5239         TRACE("[Thread %2u] jmp\n", p->thread_id);
5240
5241         thread_ip_set(t, ip->jmp.ip);
5242 }
5243
5244 static inline void
5245 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5246 {
5247         struct thread *t = &p->threads[p->thread_id];
5248         struct instruction *ip = t->ip;
5249         uint32_t header_id = ip->jmp.header_id;
5250
5251         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5252
5253         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5254 }
5255
5256 static inline void
5257 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5258 {
5259         struct thread *t = &p->threads[p->thread_id];
5260         struct instruction *ip = t->ip;
5261         uint32_t header_id = ip->jmp.header_id;
5262
5263         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5264
5265         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5266 }
5267
5268 static inline void
5269 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5270 {
5271         struct thread *t = &p->threads[p->thread_id];
5272         struct instruction *ip = t->ip;
5273         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5274
5275         TRACE("[Thread %2u] jmph\n", p->thread_id);
5276
5277         t->ip = ip_next[t->hit];
5278 }
5279
5280 static inline void
5281 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5282 {
5283         struct thread *t = &p->threads[p->thread_id];
5284         struct instruction *ip = t->ip;
5285         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5286
5287         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5288
5289         t->ip = ip_next[t->hit];
5290 }
5291
5292 static inline void
5293 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5294 {
5295         struct thread *t = &p->threads[p->thread_id];
5296         struct instruction *ip = t->ip;
5297
5298         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5299
5300         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5301 }
5302
5303 static inline void
5304 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5305 {
5306         struct thread *t = &p->threads[p->thread_id];
5307         struct instruction *ip = t->ip;
5308
5309         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5310
5311         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5312 }
5313
5314 static inline void
5315 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5316 {
5317         struct thread *t = &p->threads[p->thread_id];
5318         struct instruction *ip = t->ip;
5319
5320         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5321
5322         JMP_CMP(t, ip, ==);
5323 }
5324
5325 static inline void
5326 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5327 {
5328         struct thread *t = &p->threads[p->thread_id];
5329         struct instruction *ip = t->ip;
5330
5331         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5332
5333         JMP_CMP_MH(t, ip, ==);
5334 }
5335
5336 static inline void
5337 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5338 {
5339         struct thread *t = &p->threads[p->thread_id];
5340         struct instruction *ip = t->ip;
5341
5342         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5343
5344         JMP_CMP_HM(t, ip, ==);
5345 }
5346
5347 static inline void
5348 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5349 {
5350         struct thread *t = &p->threads[p->thread_id];
5351         struct instruction *ip = t->ip;
5352
5353         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5354
5355         JMP_CMP_HH_FAST(t, ip, ==);
5356 }
5357
5358 static inline void
5359 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5360 {
5361         struct thread *t = &p->threads[p->thread_id];
5362         struct instruction *ip = t->ip;
5363
5364         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5365
5366         JMP_CMP_I(t, ip, ==);
5367 }
5368
5369 static inline void
5370 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5371 {
5372         struct thread *t = &p->threads[p->thread_id];
5373         struct instruction *ip = t->ip;
5374
5375         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5376
5377         JMP_CMP(t, ip, !=);
5378 }
5379
5380 static inline void
5381 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5382 {
5383         struct thread *t = &p->threads[p->thread_id];
5384         struct instruction *ip = t->ip;
5385
5386         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5387
5388         JMP_CMP_MH(t, ip, !=);
5389 }
5390
5391 static inline void
5392 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5393 {
5394         struct thread *t = &p->threads[p->thread_id];
5395         struct instruction *ip = t->ip;
5396
5397         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5398
5399         JMP_CMP_HM(t, ip, !=);
5400 }
5401
5402 static inline void
5403 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5404 {
5405         struct thread *t = &p->threads[p->thread_id];
5406         struct instruction *ip = t->ip;
5407
5408         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5409
5410         JMP_CMP_HH_FAST(t, ip, !=);
5411 }
5412
5413 static inline void
5414 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5415 {
5416         struct thread *t = &p->threads[p->thread_id];
5417         struct instruction *ip = t->ip;
5418
5419         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5420
5421         JMP_CMP_I(t, ip, !=);
5422 }
5423
5424 static inline void
5425 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5426 {
5427         struct thread *t = &p->threads[p->thread_id];
5428         struct instruction *ip = t->ip;
5429
5430         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5431
5432         JMP_CMP(t, ip, <);
5433 }
5434
5435 static inline void
5436 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5437 {
5438         struct thread *t = &p->threads[p->thread_id];
5439         struct instruction *ip = t->ip;
5440
5441         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5442
5443         JMP_CMP_MH(t, ip, <);
5444 }
5445
5446 static inline void
5447 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5448 {
5449         struct thread *t = &p->threads[p->thread_id];
5450         struct instruction *ip = t->ip;
5451
5452         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5453
5454         JMP_CMP_HM(t, ip, <);
5455 }
5456
5457 static inline void
5458 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5459 {
5460         struct thread *t = &p->threads[p->thread_id];
5461         struct instruction *ip = t->ip;
5462
5463         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5464
5465         JMP_CMP_HH(t, ip, <);
5466 }
5467
5468 static inline void
5469 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5470 {
5471         struct thread *t = &p->threads[p->thread_id];
5472         struct instruction *ip = t->ip;
5473
5474         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5475
5476         JMP_CMP_MI(t, ip, <);
5477 }
5478
5479 static inline void
5480 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5481 {
5482         struct thread *t = &p->threads[p->thread_id];
5483         struct instruction *ip = t->ip;
5484
5485         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5486
5487         JMP_CMP_HI(t, ip, <);
5488 }
5489
5490 static inline void
5491 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5492 {
5493         struct thread *t = &p->threads[p->thread_id];
5494         struct instruction *ip = t->ip;
5495
5496         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5497
5498         JMP_CMP(t, ip, >);
5499 }
5500
5501 static inline void
5502 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5503 {
5504         struct thread *t = &p->threads[p->thread_id];
5505         struct instruction *ip = t->ip;
5506
5507         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5508
5509         JMP_CMP_MH(t, ip, >);
5510 }
5511
5512 static inline void
5513 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5514 {
5515         struct thread *t = &p->threads[p->thread_id];
5516         struct instruction *ip = t->ip;
5517
5518         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5519
5520         JMP_CMP_HM(t, ip, >);
5521 }
5522
5523 static inline void
5524 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5525 {
5526         struct thread *t = &p->threads[p->thread_id];
5527         struct instruction *ip = t->ip;
5528
5529         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5530
5531         JMP_CMP_HH(t, ip, >);
5532 }
5533
5534 static inline void
5535 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5536 {
5537         struct thread *t = &p->threads[p->thread_id];
5538         struct instruction *ip = t->ip;
5539
5540         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5541
5542         JMP_CMP_MI(t, ip, >);
5543 }
5544
5545 static inline void
5546 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5547 {
5548         struct thread *t = &p->threads[p->thread_id];
5549         struct instruction *ip = t->ip;
5550
5551         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5552
5553         JMP_CMP_HI(t, ip, >);
5554 }
5555
5556 /*
5557  * return.
5558  */
5559 static int
5560 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5561                        struct action *action,
5562                        char **tokens __rte_unused,
5563                        int n_tokens,
5564                        struct instruction *instr,
5565                        struct instruction_data *data __rte_unused)
5566 {
5567         CHECK(action, EINVAL);
5568         CHECK(n_tokens == 1, EINVAL);
5569
5570         instr->type = INSTR_RETURN;
5571         return 0;
5572 }
5573
5574 static inline void
5575 instr_return_exec(struct rte_swx_pipeline *p)
5576 {
5577         struct thread *t = &p->threads[p->thread_id];
5578
5579         TRACE("[Thread %2u] return\n", p->thread_id);
5580
5581         t->ip = t->ret;
5582 }
5583
5584 static int
5585 instr_translate(struct rte_swx_pipeline *p,
5586                 struct action *action,
5587                 char *string,
5588                 struct instruction *instr,
5589                 struct instruction_data *data)
5590 {
5591         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5592         int n_tokens = 0, tpos = 0;
5593
5594         /* Parse the instruction string into tokens. */
5595         for ( ; ; ) {
5596                 char *token;
5597
5598                 token = strtok_r(string, " \t\v", &string);
5599                 if (!token)
5600                         break;
5601
5602                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5603                 CHECK_NAME(token, EINVAL);
5604
5605                 tokens[n_tokens] = token;
5606                 n_tokens++;
5607         }
5608
5609         CHECK(n_tokens, EINVAL);
5610
5611         /* Handle the optional instruction label. */
5612         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5613                 strcpy(data->label, tokens[0]);
5614
5615                 tpos += 2;
5616                 CHECK(n_tokens - tpos, EINVAL);
5617         }
5618
5619         /* Identify the instruction type. */
5620         if (!strcmp(tokens[tpos], "rx"))
5621                 return instr_rx_translate(p,
5622                                           action,
5623                                           &tokens[tpos],
5624                                           n_tokens - tpos,
5625                                           instr,
5626                                           data);
5627
5628         if (!strcmp(tokens[tpos], "tx"))
5629                 return instr_tx_translate(p,
5630                                           action,
5631                                           &tokens[tpos],
5632                                           n_tokens - tpos,
5633                                           instr,
5634                                           data);
5635
5636         if (!strcmp(tokens[tpos], "drop"))
5637                 return instr_drop_translate(p,
5638                                             action,
5639                                             &tokens[tpos],
5640                                             n_tokens - tpos,
5641                                             instr,
5642                                             data);
5643
5644         if (!strcmp(tokens[tpos], "extract"))
5645                 return instr_hdr_extract_translate(p,
5646                                                    action,
5647                                                    &tokens[tpos],
5648                                                    n_tokens - tpos,
5649                                                    instr,
5650                                                    data);
5651
5652         if (!strcmp(tokens[tpos], "lookahead"))
5653                 return instr_hdr_lookahead_translate(p,
5654                                                      action,
5655                                                      &tokens[tpos],
5656                                                      n_tokens - tpos,
5657                                                      instr,
5658                                                      data);
5659
5660         if (!strcmp(tokens[tpos], "emit"))
5661                 return instr_hdr_emit_translate(p,
5662                                                 action,
5663                                                 &tokens[tpos],
5664                                                 n_tokens - tpos,
5665                                                 instr,
5666                                                 data);
5667
5668         if (!strcmp(tokens[tpos], "validate"))
5669                 return instr_hdr_validate_translate(p,
5670                                                     action,
5671                                                     &tokens[tpos],
5672                                                     n_tokens - tpos,
5673                                                     instr,
5674                                                     data);
5675
5676         if (!strcmp(tokens[tpos], "invalidate"))
5677                 return instr_hdr_invalidate_translate(p,
5678                                                       action,
5679                                                       &tokens[tpos],
5680                                                       n_tokens - tpos,
5681                                                       instr,
5682                                                       data);
5683
5684         if (!strcmp(tokens[tpos], "mov"))
5685                 return instr_mov_translate(p,
5686                                            action,
5687                                            &tokens[tpos],
5688                                            n_tokens - tpos,
5689                                            instr,
5690                                            data);
5691
5692         if (!strcmp(tokens[tpos], "add"))
5693                 return instr_alu_add_translate(p,
5694                                                action,
5695                                                &tokens[tpos],
5696                                                n_tokens - tpos,
5697                                                instr,
5698                                                data);
5699
5700         if (!strcmp(tokens[tpos], "sub"))
5701                 return instr_alu_sub_translate(p,
5702                                                action,
5703                                                &tokens[tpos],
5704                                                n_tokens - tpos,
5705                                                instr,
5706                                                data);
5707
5708         if (!strcmp(tokens[tpos], "ckadd"))
5709                 return instr_alu_ckadd_translate(p,
5710                                                  action,
5711                                                  &tokens[tpos],
5712                                                  n_tokens - tpos,
5713                                                  instr,
5714                                                  data);
5715
5716         if (!strcmp(tokens[tpos], "cksub"))
5717                 return instr_alu_cksub_translate(p,
5718                                                  action,
5719                                                  &tokens[tpos],
5720                                                  n_tokens - tpos,
5721                                                  instr,
5722                                                  data);
5723
5724         if (!strcmp(tokens[tpos], "and"))
5725                 return instr_alu_and_translate(p,
5726                                                action,
5727                                                &tokens[tpos],
5728                                                n_tokens - tpos,
5729                                                instr,
5730                                                data);
5731
5732         if (!strcmp(tokens[tpos], "or"))
5733                 return instr_alu_or_translate(p,
5734                                               action,
5735                                               &tokens[tpos],
5736                                               n_tokens - tpos,
5737                                               instr,
5738                                               data);
5739
5740         if (!strcmp(tokens[tpos], "xor"))
5741                 return instr_alu_xor_translate(p,
5742                                                action,
5743                                                &tokens[tpos],
5744                                                n_tokens - tpos,
5745                                                instr,
5746                                                data);
5747
5748         if (!strcmp(tokens[tpos], "shl"))
5749                 return instr_alu_shl_translate(p,
5750                                                action,
5751                                                &tokens[tpos],
5752                                                n_tokens - tpos,
5753                                                instr,
5754                                                data);
5755
5756         if (!strcmp(tokens[tpos], "shr"))
5757                 return instr_alu_shr_translate(p,
5758                                                action,
5759                                                &tokens[tpos],
5760                                                n_tokens - tpos,
5761                                                instr,
5762                                                data);
5763
5764         if (!strcmp(tokens[tpos], "regprefetch"))
5765                 return instr_regprefetch_translate(p,
5766                                                    action,
5767                                                    &tokens[tpos],
5768                                                    n_tokens - tpos,
5769                                                    instr,
5770                                                    data);
5771
5772         if (!strcmp(tokens[tpos], "regrd"))
5773                 return instr_regrd_translate(p,
5774                                              action,
5775                                              &tokens[tpos],
5776                                              n_tokens - tpos,
5777                                              instr,
5778                                              data);
5779
5780         if (!strcmp(tokens[tpos], "regwr"))
5781                 return instr_regwr_translate(p,
5782                                              action,
5783                                              &tokens[tpos],
5784                                              n_tokens - tpos,
5785                                              instr,
5786                                              data);
5787
5788         if (!strcmp(tokens[tpos], "regadd"))
5789                 return instr_regadd_translate(p,
5790                                               action,
5791                                               &tokens[tpos],
5792                                               n_tokens - tpos,
5793                                               instr,
5794                                               data);
5795
5796         if (!strcmp(tokens[tpos], "metprefetch"))
5797                 return instr_metprefetch_translate(p,
5798                                                    action,
5799                                                    &tokens[tpos],
5800                                                    n_tokens - tpos,
5801                                                    instr,
5802                                                    data);
5803
5804         if (!strcmp(tokens[tpos], "meter"))
5805                 return instr_meter_translate(p,
5806                                              action,
5807                                              &tokens[tpos],
5808                                              n_tokens - tpos,
5809                                              instr,
5810                                              data);
5811
5812         if (!strcmp(tokens[tpos], "table"))
5813                 return instr_table_translate(p,
5814                                              action,
5815                                              &tokens[tpos],
5816                                              n_tokens - tpos,
5817                                              instr,
5818                                              data);
5819
5820         if (!strcmp(tokens[tpos], "learn"))
5821                 return instr_learn_translate(p,
5822                                              action,
5823                                              &tokens[tpos],
5824                                              n_tokens - tpos,
5825                                              instr,
5826                                              data);
5827
5828         if (!strcmp(tokens[tpos], "forget"))
5829                 return instr_forget_translate(p,
5830                                               action,
5831                                               &tokens[tpos],
5832                                               n_tokens - tpos,
5833                                               instr,
5834                                               data);
5835
5836         if (!strcmp(tokens[tpos], "extern"))
5837                 return instr_extern_translate(p,
5838                                               action,
5839                                               &tokens[tpos],
5840                                               n_tokens - tpos,
5841                                               instr,
5842                                               data);
5843
5844         if (!strcmp(tokens[tpos], "jmp"))
5845                 return instr_jmp_translate(p,
5846                                            action,
5847                                            &tokens[tpos],
5848                                            n_tokens - tpos,
5849                                            instr,
5850                                            data);
5851
5852         if (!strcmp(tokens[tpos], "jmpv"))
5853                 return instr_jmp_valid_translate(p,
5854                                                  action,
5855                                                  &tokens[tpos],
5856                                                  n_tokens - tpos,
5857                                                  instr,
5858                                                  data);
5859
5860         if (!strcmp(tokens[tpos], "jmpnv"))
5861                 return instr_jmp_invalid_translate(p,
5862                                                    action,
5863                                                    &tokens[tpos],
5864                                                    n_tokens - tpos,
5865                                                    instr,
5866                                                    data);
5867
5868         if (!strcmp(tokens[tpos], "jmph"))
5869                 return instr_jmp_hit_translate(p,
5870                                                action,
5871                                                &tokens[tpos],
5872                                                n_tokens - tpos,
5873                                                instr,
5874                                                data);
5875
5876         if (!strcmp(tokens[tpos], "jmpnh"))
5877                 return instr_jmp_miss_translate(p,
5878                                                 action,
5879                                                 &tokens[tpos],
5880                                                 n_tokens - tpos,
5881                                                 instr,
5882                                                 data);
5883
5884         if (!strcmp(tokens[tpos], "jmpa"))
5885                 return instr_jmp_action_hit_translate(p,
5886                                                       action,
5887                                                       &tokens[tpos],
5888                                                       n_tokens - tpos,
5889                                                       instr,
5890                                                       data);
5891
5892         if (!strcmp(tokens[tpos], "jmpna"))
5893                 return instr_jmp_action_miss_translate(p,
5894                                                        action,
5895                                                        &tokens[tpos],
5896                                                        n_tokens - tpos,
5897                                                        instr,
5898                                                        data);
5899
5900         if (!strcmp(tokens[tpos], "jmpeq"))
5901                 return instr_jmp_eq_translate(p,
5902                                               action,
5903                                               &tokens[tpos],
5904                                               n_tokens - tpos,
5905                                               instr,
5906                                               data);
5907
5908         if (!strcmp(tokens[tpos], "jmpneq"))
5909                 return instr_jmp_neq_translate(p,
5910                                                action,
5911                                                &tokens[tpos],
5912                                                n_tokens - tpos,
5913                                                instr,
5914                                                data);
5915
5916         if (!strcmp(tokens[tpos], "jmplt"))
5917                 return instr_jmp_lt_translate(p,
5918                                               action,
5919                                               &tokens[tpos],
5920                                               n_tokens - tpos,
5921                                               instr,
5922                                               data);
5923
5924         if (!strcmp(tokens[tpos], "jmpgt"))
5925                 return instr_jmp_gt_translate(p,
5926                                               action,
5927                                               &tokens[tpos],
5928                                               n_tokens - tpos,
5929                                               instr,
5930                                               data);
5931
5932         if (!strcmp(tokens[tpos], "return"))
5933                 return instr_return_translate(p,
5934                                               action,
5935                                               &tokens[tpos],
5936                                               n_tokens - tpos,
5937                                               instr,
5938                                               data);
5939
5940         return -EINVAL;
5941 }
5942
5943 static struct instruction_data *
5944 label_find(struct instruction_data *data, uint32_t n, const char *label)
5945 {
5946         uint32_t i;
5947
5948         for (i = 0; i < n; i++)
5949                 if (!strcmp(label, data[i].label))
5950                         return &data[i];
5951
5952         return NULL;
5953 }
5954
5955 static uint32_t
5956 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5957 {
5958         uint32_t count = 0, i;
5959
5960         if (!label[0])
5961                 return 0;
5962
5963         for (i = 0; i < n; i++)
5964                 if (!strcmp(label, data[i].jmp_label))
5965                         count++;
5966
5967         return count;
5968 }
5969
5970 static int
5971 instr_label_check(struct instruction_data *instruction_data,
5972                   uint32_t n_instructions)
5973 {
5974         uint32_t i;
5975
5976         /* Check that all instruction labels are unique. */
5977         for (i = 0; i < n_instructions; i++) {
5978                 struct instruction_data *data = &instruction_data[i];
5979                 char *label = data->label;
5980                 uint32_t j;
5981
5982                 if (!label[0])
5983                         continue;
5984
5985                 for (j = i + 1; j < n_instructions; j++)
5986                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
5987         }
5988
5989         /* Get users for each instruction label. */
5990         for (i = 0; i < n_instructions; i++) {
5991                 struct instruction_data *data = &instruction_data[i];
5992                 char *label = data->label;
5993
5994                 data->n_users = label_is_used(instruction_data,
5995                                               n_instructions,
5996                                               label);
5997         }
5998
5999         return 0;
6000 }
6001
6002 static int
6003 instr_jmp_resolve(struct instruction *instructions,
6004                   struct instruction_data *instruction_data,
6005                   uint32_t n_instructions)
6006 {
6007         uint32_t i;
6008
6009         for (i = 0; i < n_instructions; i++) {
6010                 struct instruction *instr = &instructions[i];
6011                 struct instruction_data *data = &instruction_data[i];
6012                 struct instruction_data *found;
6013
6014                 if (!instruction_is_jmp(instr))
6015                         continue;
6016
6017                 found = label_find(instruction_data,
6018                                    n_instructions,
6019                                    data->jmp_label);
6020                 CHECK(found, EINVAL);
6021
6022                 instr->jmp.ip = &instructions[found - instruction_data];
6023         }
6024
6025         return 0;
6026 }
6027
6028 static int
6029 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6030              struct action *a,
6031              struct instruction *instr,
6032              struct instruction_data *data __rte_unused,
6033              uint32_t n_instructions)
6034 {
6035         if (!a) {
6036                 enum instruction_type type;
6037                 uint32_t i;
6038
6039                 /* Check that the first instruction is rx. */
6040                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6041
6042                 /* Check that there is at least one tx instruction. */
6043                 for (i = 0; i < n_instructions; i++) {
6044                         type = instr[i].type;
6045
6046                         if (instruction_is_tx(type))
6047                                 break;
6048                 }
6049                 CHECK(i < n_instructions, EINVAL);
6050
6051                 /* Check that the last instruction is either tx or unconditional
6052                  * jump.
6053                  */
6054                 type = instr[n_instructions - 1].type;
6055                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6056         }
6057
6058         if (a) {
6059                 enum instruction_type type;
6060                 uint32_t i;
6061
6062                 /* Check that there is at least one return or tx instruction. */
6063                 for (i = 0; i < n_instructions; i++) {
6064                         type = instr[i].type;
6065
6066                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6067                                 break;
6068                 }
6069                 CHECK(i < n_instructions, EINVAL);
6070         }
6071
6072         return 0;
6073 }
6074
6075 static uint32_t
6076 instr_compact(struct instruction *instructions,
6077               struct instruction_data *instruction_data,
6078               uint32_t n_instructions)
6079 {
6080         uint32_t i, pos = 0;
6081
6082         /* Eliminate the invalid instructions that have been optimized out. */
6083         for (i = 0; i < n_instructions; i++) {
6084                 struct instruction *instr = &instructions[i];
6085                 struct instruction_data *data = &instruction_data[i];
6086
6087                 if (data->invalid)
6088                         continue;
6089
6090                 if (i != pos) {
6091                         memcpy(&instructions[pos], instr, sizeof(*instr));
6092                         memcpy(&instruction_data[pos], data, sizeof(*data));
6093                 }
6094
6095                 pos++;
6096         }
6097
6098         return pos;
6099 }
6100
6101 static int
6102 instr_pattern_extract_many_search(struct instruction *instr,
6103                                   struct instruction_data *data,
6104                                   uint32_t n_instr,
6105                                   uint32_t *n_pattern_instr)
6106 {
6107         uint32_t i;
6108
6109         for (i = 0; i < n_instr; i++) {
6110                 if (data[i].invalid)
6111                         break;
6112
6113                 if (instr[i].type != INSTR_HDR_EXTRACT)
6114                         break;
6115
6116                 if (i == RTE_DIM(instr->io.hdr.header_id))
6117                         break;
6118
6119                 if (i && data[i].n_users)
6120                         break;
6121         }
6122
6123         if (i < 2)
6124                 return 0;
6125
6126         *n_pattern_instr = i;
6127         return 1;
6128 }
6129
6130 static void
6131 instr_pattern_extract_many_replace(struct instruction *instr,
6132                                    struct instruction_data *data,
6133                                    uint32_t n_instr)
6134 {
6135         uint32_t i;
6136
6137         for (i = 1; i < n_instr; i++) {
6138                 instr[0].type++;
6139                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6140                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6141                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6142
6143                 data[i].invalid = 1;
6144         }
6145 }
6146
6147 static uint32_t
6148 instr_pattern_extract_many_optimize(struct instruction *instructions,
6149                                     struct instruction_data *instruction_data,
6150                                     uint32_t n_instructions)
6151 {
6152         uint32_t i;
6153
6154         for (i = 0; i < n_instructions; ) {
6155                 struct instruction *instr = &instructions[i];
6156                 struct instruction_data *data = &instruction_data[i];
6157                 uint32_t n_instr = 0;
6158                 int detected;
6159
6160                 /* Extract many. */
6161                 detected = instr_pattern_extract_many_search(instr,
6162                                                              data,
6163                                                              n_instructions - i,
6164                                                              &n_instr);
6165                 if (detected) {
6166                         instr_pattern_extract_many_replace(instr,
6167                                                            data,
6168                                                            n_instr);
6169                         i += n_instr;
6170                         continue;
6171                 }
6172
6173                 /* No pattern starting at the current instruction. */
6174                 i++;
6175         }
6176
6177         /* Eliminate the invalid instructions that have been optimized out. */
6178         n_instructions = instr_compact(instructions,
6179                                        instruction_data,
6180                                        n_instructions);
6181
6182         return n_instructions;
6183 }
6184
6185 static int
6186 instr_pattern_emit_many_tx_search(struct instruction *instr,
6187                                   struct instruction_data *data,
6188                                   uint32_t n_instr,
6189                                   uint32_t *n_pattern_instr)
6190 {
6191         uint32_t i;
6192
6193         for (i = 0; i < n_instr; i++) {
6194                 if (data[i].invalid)
6195                         break;
6196
6197                 if (instr[i].type != INSTR_HDR_EMIT)
6198                         break;
6199
6200                 if (i == RTE_DIM(instr->io.hdr.header_id))
6201                         break;
6202
6203                 if (i && data[i].n_users)
6204                         break;
6205         }
6206
6207         if (!i)
6208                 return 0;
6209
6210         if (!instruction_is_tx(instr[i].type))
6211                 return 0;
6212
6213         if (data[i].n_users)
6214                 return 0;
6215
6216         i++;
6217
6218         *n_pattern_instr = i;
6219         return 1;
6220 }
6221
6222 static void
6223 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6224                                    struct instruction_data *data,
6225                                    uint32_t n_instr)
6226 {
6227         uint32_t i;
6228
6229         /* Any emit instruction in addition to the first one. */
6230         for (i = 1; i < n_instr - 1; i++) {
6231                 instr[0].type++;
6232                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6233                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6234                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6235
6236                 data[i].invalid = 1;
6237         }
6238
6239         /* The TX instruction is the last one in the pattern. */
6240         instr[0].type++;
6241         instr[0].io.io.offset = instr[i].io.io.offset;
6242         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6243         data[i].invalid = 1;
6244 }
6245
6246 static uint32_t
6247 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6248                                     struct instruction_data *instruction_data,
6249                                     uint32_t n_instructions)
6250 {
6251         uint32_t i;
6252
6253         for (i = 0; i < n_instructions; ) {
6254                 struct instruction *instr = &instructions[i];
6255                 struct instruction_data *data = &instruction_data[i];
6256                 uint32_t n_instr = 0;
6257                 int detected;
6258
6259                 /* Emit many + TX. */
6260                 detected = instr_pattern_emit_many_tx_search(instr,
6261                                                              data,
6262                                                              n_instructions - i,
6263                                                              &n_instr);
6264                 if (detected) {
6265                         instr_pattern_emit_many_tx_replace(instr,
6266                                                            data,
6267                                                            n_instr);
6268                         i += n_instr;
6269                         continue;
6270                 }
6271
6272                 /* No pattern starting at the current instruction. */
6273                 i++;
6274         }
6275
6276         /* Eliminate the invalid instructions that have been optimized out. */
6277         n_instructions = instr_compact(instructions,
6278                                        instruction_data,
6279                                        n_instructions);
6280
6281         return n_instructions;
6282 }
6283
6284 static uint32_t
6285 action_arg_src_mov_count(struct action *a,
6286                          uint32_t arg_id,
6287                          struct instruction *instructions,
6288                          struct instruction_data *instruction_data,
6289                          uint32_t n_instructions);
6290
6291 static int
6292 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6293                                       struct action *a,
6294                                       struct instruction *instr,
6295                                       struct instruction_data *data,
6296                                       uint32_t n_instr,
6297                                       struct instruction *instructions,
6298                                       struct instruction_data *instruction_data,
6299                                       uint32_t n_instructions,
6300                                       uint32_t *n_pattern_instr)
6301 {
6302         struct header *h;
6303         uint32_t src_field_id, i, j;
6304
6305         /* Prerequisites. */
6306         if (!a || !a->st)
6307                 return 0;
6308
6309         /* First instruction: MOV_HM. */
6310         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6311                 return 0;
6312
6313         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6314         if (!h || h->st->var_size)
6315                 return 0;
6316
6317         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6318                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6319                         break;
6320
6321         if (src_field_id == a->st->n_fields)
6322                 return 0;
6323
6324         if (instr[0].mov.dst.offset ||
6325             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6326             instr[0].mov.src.struct_id ||
6327             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6328             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6329                 return 0;
6330
6331         if ((n_instr < h->st->n_fields + 1) ||
6332              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6333                 return 0;
6334
6335         /* Subsequent instructions: MOV_HM. */
6336         for (i = 1; i < h->st->n_fields; i++)
6337                 if (data[i].invalid ||
6338                     data[i].n_users ||
6339                     (instr[i].type != INSTR_MOV_HM) ||
6340                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6341                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6342                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6343                     instr[i].mov.src.struct_id ||
6344                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6345                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6346                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6347                         return 0;
6348
6349         /* Last instruction: HDR_VALIDATE. */
6350         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6351             (instr[i].valid.header_id != h->id))
6352                 return 0;
6353
6354         /* Check that none of the action args that are used as source for this
6355          * DMA transfer are not used as source in any other mov instruction.
6356          */
6357         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6358                 uint32_t n_users;
6359
6360                 n_users = action_arg_src_mov_count(a,
6361                                                    j,
6362                                                    instructions,
6363                                                    instruction_data,
6364                                                    n_instructions);
6365                 if (n_users > 1)
6366                         return 0;
6367         }
6368
6369         *n_pattern_instr = 1 + i;
6370         return 1;
6371 }
6372
6373 static void
6374 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6375                                        struct action *a,
6376                                        struct instruction *instr,
6377                                        struct instruction_data *data,
6378                                        uint32_t n_instr)
6379 {
6380         struct header *h;
6381         uint32_t src_field_id, src_offset, i;
6382
6383         /* Read from the instructions before they are modified. */
6384         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6385         if (!h)
6386                 return;
6387
6388         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6389                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6390                         break;
6391
6392         if (src_field_id == a->st->n_fields)
6393                 return;
6394
6395         src_offset = instr[0].mov.src.offset;
6396
6397         /* Modify the instructions. */
6398         instr[0].type = INSTR_DMA_HT;
6399         instr[0].dma.dst.header_id[0] = h->id;
6400         instr[0].dma.dst.struct_id[0] = h->struct_id;
6401         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6402         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6403
6404         for (i = 1; i < n_instr; i++)
6405                 data[i].invalid = 1;
6406
6407         /* Update the endianness of the action arguments to header endianness. */
6408         for (i = 0; i < h->st->n_fields; i++)
6409                 a->args_endianness[src_field_id + i] = 1;
6410 }
6411
6412 static uint32_t
6413 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6414                                         struct action *a,
6415                                         struct instruction *instructions,
6416                                         struct instruction_data *instruction_data,
6417                                         uint32_t n_instructions)
6418 {
6419         uint32_t i;
6420
6421         if (!a || !a->st)
6422                 return n_instructions;
6423
6424         for (i = 0; i < n_instructions; ) {
6425                 struct instruction *instr = &instructions[i];
6426                 struct instruction_data *data = &instruction_data[i];
6427                 uint32_t n_instr = 0;
6428                 int detected;
6429
6430                 /* Mov all + validate. */
6431                 detected = instr_pattern_mov_all_validate_search(p,
6432                                                                  a,
6433                                                                  instr,
6434                                                                  data,
6435                                                                  n_instructions - i,
6436                                                                  instructions,
6437                                                                  instruction_data,
6438                                                                  n_instructions,
6439                                                                  &n_instr);
6440                 if (detected) {
6441                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6442                         i += n_instr;
6443                         continue;
6444                 }
6445
6446                 /* No pattern starting at the current instruction. */
6447                 i++;
6448         }
6449
6450         /* Eliminate the invalid instructions that have been optimized out. */
6451         n_instructions = instr_compact(instructions,
6452                                        instruction_data,
6453                                        n_instructions);
6454
6455         return n_instructions;
6456 }
6457
6458 static int
6459 instr_pattern_dma_many_search(struct instruction *instr,
6460                               struct instruction_data *data,
6461                               uint32_t n_instr,
6462                               uint32_t *n_pattern_instr)
6463 {
6464         uint32_t i;
6465
6466         for (i = 0; i < n_instr; i++) {
6467                 if (data[i].invalid)
6468                         break;
6469
6470                 if (instr[i].type != INSTR_DMA_HT)
6471                         break;
6472
6473                 if (i == RTE_DIM(instr->dma.dst.header_id))
6474                         break;
6475
6476                 if (i && data[i].n_users)
6477                         break;
6478         }
6479
6480         if (i < 2)
6481                 return 0;
6482
6483         *n_pattern_instr = i;
6484         return 1;
6485 }
6486
6487 static void
6488 instr_pattern_dma_many_replace(struct instruction *instr,
6489                                struct instruction_data *data,
6490                                uint32_t n_instr)
6491 {
6492         uint32_t i;
6493
6494         for (i = 1; i < n_instr; i++) {
6495                 instr[0].type++;
6496                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6497                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6498                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6499                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6500
6501                 data[i].invalid = 1;
6502         }
6503 }
6504
6505 static uint32_t
6506 instr_pattern_dma_many_optimize(struct instruction *instructions,
6507                struct instruction_data *instruction_data,
6508                uint32_t n_instructions)
6509 {
6510         uint32_t i;
6511
6512         for (i = 0; i < n_instructions; ) {
6513                 struct instruction *instr = &instructions[i];
6514                 struct instruction_data *data = &instruction_data[i];
6515                 uint32_t n_instr = 0;
6516                 int detected;
6517
6518                 /* DMA many. */
6519                 detected = instr_pattern_dma_many_search(instr,
6520                                                          data,
6521                                                          n_instructions - i,
6522                                                          &n_instr);
6523                 if (detected) {
6524                         instr_pattern_dma_many_replace(instr, data, n_instr);
6525                         i += n_instr;
6526                         continue;
6527                 }
6528
6529                 /* No pattern starting at the current instruction. */
6530                 i++;
6531         }
6532
6533         /* Eliminate the invalid instructions that have been optimized out. */
6534         n_instructions = instr_compact(instructions,
6535                                        instruction_data,
6536                                        n_instructions);
6537
6538         return n_instructions;
6539 }
6540
6541 static uint32_t
6542 instr_optimize(struct rte_swx_pipeline *p,
6543                struct action *a,
6544                struct instruction *instructions,
6545                struct instruction_data *instruction_data,
6546                uint32_t n_instructions)
6547 {
6548         /* Extract many. */
6549         n_instructions = instr_pattern_extract_many_optimize(instructions,
6550                                                              instruction_data,
6551                                                              n_instructions);
6552
6553         /* Emit many + TX. */
6554         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6555                                                              instruction_data,
6556                                                              n_instructions);
6557
6558         /* Mov all + validate. */
6559         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6560                                                                  a,
6561                                                                  instructions,
6562                                                                  instruction_data,
6563                                                                  n_instructions);
6564
6565         /* DMA many. */
6566         n_instructions = instr_pattern_dma_many_optimize(instructions,
6567                                                          instruction_data,
6568                                                          n_instructions);
6569
6570         return n_instructions;
6571 }
6572
6573 static int
6574 instruction_config(struct rte_swx_pipeline *p,
6575                    struct action *a,
6576                    const char **instructions,
6577                    uint32_t n_instructions)
6578 {
6579         struct instruction *instr = NULL;
6580         struct instruction_data *data = NULL;
6581         int err = 0;
6582         uint32_t i;
6583
6584         CHECK(n_instructions, EINVAL);
6585         CHECK(instructions, EINVAL);
6586         for (i = 0; i < n_instructions; i++)
6587                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6588
6589         /* Memory allocation. */
6590         instr = calloc(n_instructions, sizeof(struct instruction));
6591         if (!instr) {
6592                 err = -ENOMEM;
6593                 goto error;
6594         }
6595
6596         data = calloc(n_instructions, sizeof(struct instruction_data));
6597         if (!data) {
6598                 err = -ENOMEM;
6599                 goto error;
6600         }
6601
6602         for (i = 0; i < n_instructions; i++) {
6603                 char *string = strdup(instructions[i]);
6604                 if (!string) {
6605                         err = -ENOMEM;
6606                         goto error;
6607                 }
6608
6609                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6610                 if (err) {
6611                         free(string);
6612                         goto error;
6613                 }
6614
6615                 free(string);
6616         }
6617
6618         err = instr_label_check(data, n_instructions);
6619         if (err)
6620                 goto error;
6621
6622         err = instr_verify(p, a, instr, data, n_instructions);
6623         if (err)
6624                 goto error;
6625
6626         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6627
6628         err = instr_jmp_resolve(instr, data, n_instructions);
6629         if (err)
6630                 goto error;
6631
6632         if (a) {
6633                 a->instructions = instr;
6634                 a->instruction_data = data;
6635                 a->n_instructions = n_instructions;
6636         } else {
6637                 p->instructions = instr;
6638                 p->instruction_data = data;
6639                 p->n_instructions = n_instructions;
6640         }
6641
6642         return 0;
6643
6644 error:
6645         free(data);
6646         free(instr);
6647         return err;
6648 }
6649
6650 static instr_exec_t instruction_table[] = {
6651         [INSTR_RX] = instr_rx_exec,
6652         [INSTR_TX] = instr_tx_exec,
6653         [INSTR_TX_I] = instr_tx_i_exec,
6654
6655         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6656         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6657         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6658         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6659         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6660         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6661         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6662         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6663         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6664         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6665
6666         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6667         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6668         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6669         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6670         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6671         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6672         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6673         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6674         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6675
6676         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6677         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6678
6679         [INSTR_MOV] = instr_mov_exec,
6680         [INSTR_MOV_MH] = instr_mov_mh_exec,
6681         [INSTR_MOV_HM] = instr_mov_hm_exec,
6682         [INSTR_MOV_HH] = instr_mov_hh_exec,
6683         [INSTR_MOV_I] = instr_mov_i_exec,
6684
6685         [INSTR_DMA_HT] = instr_dma_ht_exec,
6686         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6687         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6688         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6689         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6690         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6691         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6692         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6693
6694         [INSTR_ALU_ADD] = instr_alu_add_exec,
6695         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6696         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6697         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6698         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6699         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6700
6701         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6702         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6703         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6704         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6705         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6706         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6707
6708         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6709         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6710         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6711         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6712
6713         [INSTR_ALU_AND] = instr_alu_and_exec,
6714         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6715         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6716         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6717         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6718
6719         [INSTR_ALU_OR] = instr_alu_or_exec,
6720         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6721         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6722         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6723         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6724
6725         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6726         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6727         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6728         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6729         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6730
6731         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6732         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6733         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6734         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6735         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6736         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6737
6738         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6739         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6740         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6741         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6742         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6743         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6744
6745         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6746         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6747         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6748
6749         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6750         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6751         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6752         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6753         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6754         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6755
6756         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6757         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6758         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6759         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6760         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6761         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6762         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6763         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6764         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6765
6766         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6767         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6768         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6769         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6770         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6771         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6772         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6773         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6774         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6775
6776         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6777         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6778         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6779
6780         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6781         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6782         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6783         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6784         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6785         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6786         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6787         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6788         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6789         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6790         [INSTR_METER_IMM] = instr_meter_imm_exec,
6791         [INSTR_METER_IMI] = instr_meter_imi_exec,
6792
6793         [INSTR_TABLE] = instr_table_exec,
6794         [INSTR_TABLE_AF] = instr_table_af_exec,
6795         [INSTR_SELECTOR] = instr_selector_exec,
6796         [INSTR_LEARNER] = instr_learner_exec,
6797         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6798         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6799         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6800         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6801         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6802
6803         [INSTR_JMP] = instr_jmp_exec,
6804         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6805         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6806         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6807         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6808         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6809         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6810
6811         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6812         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6813         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6814         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6815         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6816
6817         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6818         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6819         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6820         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6821         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6822
6823         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6824         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6825         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6826         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6827         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6828         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6829
6830         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6831         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6832         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6833         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6834         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6835         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6836
6837         [INSTR_RETURN] = instr_return_exec,
6838 };
6839
6840 static int
6841 instruction_table_build(struct rte_swx_pipeline *p)
6842 {
6843         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6844                                       sizeof(struct instr_exec_t *));
6845         if (!p->instruction_table)
6846                 return -EINVAL;
6847
6848         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6849
6850         return 0;
6851 }
6852
6853 static void
6854 instruction_table_build_free(struct rte_swx_pipeline *p)
6855 {
6856         if (!p->instruction_table)
6857                 return;
6858
6859         free(p->instruction_table);
6860         p->instruction_table = NULL;
6861 }
6862
6863 static void
6864 instruction_table_free(struct rte_swx_pipeline *p)
6865 {
6866         instruction_table_build_free(p);
6867 }
6868
6869 static inline void
6870 instr_exec(struct rte_swx_pipeline *p)
6871 {
6872         struct thread *t = &p->threads[p->thread_id];
6873         struct instruction *ip = t->ip;
6874         instr_exec_t instr = p->instruction_table[ip->type];
6875
6876         instr(p);
6877 }
6878
6879 /*
6880  * Action.
6881  */
6882 static struct action *
6883 action_find(struct rte_swx_pipeline *p, const char *name)
6884 {
6885         struct action *elem;
6886
6887         if (!name)
6888                 return NULL;
6889
6890         TAILQ_FOREACH(elem, &p->actions, node)
6891                 if (strcmp(elem->name, name) == 0)
6892                         return elem;
6893
6894         return NULL;
6895 }
6896
6897 static struct action *
6898 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6899 {
6900         struct action *action = NULL;
6901
6902         TAILQ_FOREACH(action, &p->actions, node)
6903                 if (action->id == id)
6904                         return action;
6905
6906         return NULL;
6907 }
6908
6909 static struct field *
6910 action_field_find(struct action *a, const char *name)
6911 {
6912         return a->st ? struct_type_field_find(a->st, name) : NULL;
6913 }
6914
6915 static struct field *
6916 action_field_parse(struct action *action, const char *name)
6917 {
6918         if (name[0] != 't' || name[1] != '.')
6919                 return NULL;
6920
6921         return action_field_find(action, &name[2]);
6922 }
6923
6924 static int
6925 action_has_nbo_args(struct action *a)
6926 {
6927         uint32_t i;
6928
6929         /* Return if the action does not have any args. */
6930         if (!a->st)
6931                 return 0; /* FALSE */
6932
6933         for (i = 0; i < a->st->n_fields; i++)
6934                 if (a->args_endianness[i])
6935                         return 1; /* TRUE */
6936
6937         return 0; /* FALSE */
6938 }
6939
6940 static int
6941 action_does_learning(struct action *a)
6942 {
6943         uint32_t i;
6944
6945         for (i = 0; i < a->n_instructions; i++)
6946                 switch (a->instructions[i].type) {
6947                 case INSTR_LEARNER_LEARN:
6948                         return 1; /* TRUE */
6949
6950                 case INSTR_LEARNER_FORGET:
6951                         return 1; /* TRUE */
6952
6953                 default:
6954                         continue;
6955                 }
6956
6957         return 0; /* FALSE */
6958 }
6959
6960 int
6961 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6962                                const char *name,
6963                                const char *args_struct_type_name,
6964                                const char **instructions,
6965                                uint32_t n_instructions)
6966 {
6967         struct struct_type *args_struct_type = NULL;
6968         struct action *a;
6969         int err;
6970
6971         CHECK(p, EINVAL);
6972
6973         CHECK_NAME(name, EINVAL);
6974         CHECK(!action_find(p, name), EEXIST);
6975
6976         if (args_struct_type_name) {
6977                 CHECK_NAME(args_struct_type_name, EINVAL);
6978                 args_struct_type = struct_type_find(p, args_struct_type_name);
6979                 CHECK(args_struct_type, EINVAL);
6980                 CHECK(!args_struct_type->var_size, EINVAL);
6981         }
6982
6983         /* Node allocation. */
6984         a = calloc(1, sizeof(struct action));
6985         CHECK(a, ENOMEM);
6986         if (args_struct_type) {
6987                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6988                 if (!a->args_endianness) {
6989                         free(a);
6990                         CHECK(0, ENOMEM);
6991                 }
6992         }
6993
6994         /* Node initialization. */
6995         strcpy(a->name, name);
6996         a->st = args_struct_type;
6997         a->id = p->n_actions;
6998
6999         /* Instruction translation. */
7000         err = instruction_config(p, a, instructions, n_instructions);
7001         if (err) {
7002                 free(a->args_endianness);
7003                 free(a);
7004                 return err;
7005         }
7006
7007         /* Node add to tailq. */
7008         TAILQ_INSERT_TAIL(&p->actions, a, node);
7009         p->n_actions++;
7010
7011         return 0;
7012 }
7013
7014 static int
7015 action_build(struct rte_swx_pipeline *p)
7016 {
7017         struct action *action;
7018
7019         /* p->action_instructions. */
7020         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7021         CHECK(p->action_instructions, ENOMEM);
7022
7023         TAILQ_FOREACH(action, &p->actions, node)
7024                 p->action_instructions[action->id] = action->instructions;
7025
7026         /* p->action_funcs. */
7027         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7028         CHECK(p->action_funcs, ENOMEM);
7029
7030         return 0;
7031 }
7032
7033 static void
7034 action_build_free(struct rte_swx_pipeline *p)
7035 {
7036         free(p->action_funcs);
7037         p->action_funcs = NULL;
7038
7039         free(p->action_instructions);
7040         p->action_instructions = NULL;
7041 }
7042
7043 static void
7044 action_free(struct rte_swx_pipeline *p)
7045 {
7046         action_build_free(p);
7047
7048         for ( ; ; ) {
7049                 struct action *action;
7050
7051                 action = TAILQ_FIRST(&p->actions);
7052                 if (!action)
7053                         break;
7054
7055                 TAILQ_REMOVE(&p->actions, action, node);
7056                 free(action->instruction_data);
7057                 free(action->instructions);
7058                 free(action);
7059         }
7060 }
7061
7062 static uint32_t
7063 action_arg_src_mov_count(struct action *a,
7064                          uint32_t arg_id,
7065                          struct instruction *instructions,
7066                          struct instruction_data *instruction_data,
7067                          uint32_t n_instructions)
7068 {
7069         uint32_t offset, n_users = 0, i;
7070
7071         if (!a->st ||
7072             (arg_id >= a->st->n_fields) ||
7073             !instructions ||
7074             !instruction_data ||
7075             !n_instructions)
7076                 return 0;
7077
7078         offset = a->st->fields[arg_id].offset / 8;
7079
7080         for (i = 0; i < n_instructions; i++) {
7081                 struct instruction *instr = &instructions[i];
7082                 struct instruction_data *data = &instruction_data[i];
7083
7084                 if (data->invalid ||
7085                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7086                     instr->mov.src.struct_id ||
7087                     (instr->mov.src.offset != offset))
7088                         continue;
7089
7090                 n_users++;
7091         }
7092
7093         return n_users;
7094 }
7095
7096 /*
7097  * Table.
7098  */
7099 static struct table_type *
7100 table_type_find(struct rte_swx_pipeline *p, const char *name)
7101 {
7102         struct table_type *elem;
7103
7104         TAILQ_FOREACH(elem, &p->table_types, node)
7105                 if (strcmp(elem->name, name) == 0)
7106                         return elem;
7107
7108         return NULL;
7109 }
7110
7111 static struct table_type *
7112 table_type_resolve(struct rte_swx_pipeline *p,
7113                    const char *recommended_type_name,
7114                    enum rte_swx_table_match_type match_type)
7115 {
7116         struct table_type *elem;
7117
7118         /* Only consider the recommended type if the match type is correct. */
7119         if (recommended_type_name)
7120                 TAILQ_FOREACH(elem, &p->table_types, node)
7121                         if (!strcmp(elem->name, recommended_type_name) &&
7122                             (elem->match_type == match_type))
7123                                 return elem;
7124
7125         /* Ignore the recommended type and get the first element with this match
7126          * type.
7127          */
7128         TAILQ_FOREACH(elem, &p->table_types, node)
7129                 if (elem->match_type == match_type)
7130                         return elem;
7131
7132         return NULL;
7133 }
7134
7135 static struct table *
7136 table_find(struct rte_swx_pipeline *p, const char *name)
7137 {
7138         struct table *elem;
7139
7140         TAILQ_FOREACH(elem, &p->tables, node)
7141                 if (strcmp(elem->name, name) == 0)
7142                         return elem;
7143
7144         return NULL;
7145 }
7146
7147 static struct table *
7148 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7149 {
7150         struct table *table = NULL;
7151
7152         TAILQ_FOREACH(table, &p->tables, node)
7153                 if (table->id == id)
7154                         return table;
7155
7156         return NULL;
7157 }
7158
7159 int
7160 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7161                                      const char *name,
7162                                      enum rte_swx_table_match_type match_type,
7163                                      struct rte_swx_table_ops *ops)
7164 {
7165         struct table_type *elem;
7166
7167         CHECK(p, EINVAL);
7168
7169         CHECK_NAME(name, EINVAL);
7170         CHECK(!table_type_find(p, name), EEXIST);
7171
7172         CHECK(ops, EINVAL);
7173         CHECK(ops->create, EINVAL);
7174         CHECK(ops->lkp, EINVAL);
7175         CHECK(ops->free, EINVAL);
7176
7177         /* Node allocation. */
7178         elem = calloc(1, sizeof(struct table_type));
7179         CHECK(elem, ENOMEM);
7180
7181         /* Node initialization. */
7182         strcpy(elem->name, name);
7183         elem->match_type = match_type;
7184         memcpy(&elem->ops, ops, sizeof(*ops));
7185
7186         /* Node add to tailq. */
7187         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7188
7189         return 0;
7190 }
7191
7192 static int
7193 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7194                          uint32_t n_fields,
7195                          enum rte_swx_table_match_type *match_type)
7196 {
7197         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7198
7199         for (i = 0; i < n_fields; i++) {
7200                 struct rte_swx_match_field_params  *f = &fields[i];
7201
7202                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7203                         n_fields_em++;
7204
7205                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7206                         n_fields_lpm++;
7207         }
7208
7209         if ((n_fields_lpm > 1) ||
7210             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7211                 return -EINVAL;
7212
7213         *match_type = (n_fields_em == n_fields) ?
7214                        RTE_SWX_TABLE_MATCH_EXACT :
7215                        RTE_SWX_TABLE_MATCH_WILDCARD;
7216
7217         return 0;
7218 }
7219
7220 static int
7221 table_match_fields_check(struct rte_swx_pipeline *p,
7222                          struct rte_swx_pipeline_table_params *params,
7223                          struct header **header)
7224 {
7225         struct header *h0 = NULL;
7226         struct field *hf, *mf;
7227         uint32_t *offset = NULL, i;
7228         int status = 0;
7229
7230         /* Return if no match fields. */
7231         if (!params->n_fields) {
7232                 if (params->fields) {
7233                         status = -EINVAL;
7234                         goto end;
7235                 }
7236
7237                 if (header)
7238                         *header = NULL;
7239
7240                 return 0;
7241         }
7242
7243         /* Memory allocation. */
7244         offset = calloc(params->n_fields, sizeof(uint32_t));
7245         if (!offset) {
7246                 status = -ENOMEM;
7247                 goto end;
7248         }
7249
7250         /* Check that all the match fields belong to either the same header or
7251          * to the meta-data.
7252          */
7253         hf = header_field_parse(p, params->fields[0].name, &h0);
7254         mf = metadata_field_parse(p, params->fields[0].name);
7255         if ((!hf && !mf) || (hf && hf->var_size)) {
7256                 status = -EINVAL;
7257                 goto end;
7258         }
7259
7260         offset[0] = h0 ? hf->offset : mf->offset;
7261
7262         for (i = 1; i < params->n_fields; i++)
7263                 if (h0) {
7264                         struct header *h;
7265
7266                         hf = header_field_parse(p, params->fields[i].name, &h);
7267                         if (!hf || (h->id != h0->id) || hf->var_size) {
7268                                 status = -EINVAL;
7269                                 goto end;
7270                         }
7271
7272                         offset[i] = hf->offset;
7273                 } else {
7274                         mf = metadata_field_parse(p, params->fields[i].name);
7275                         if (!mf) {
7276                                 status = -EINVAL;
7277                                 goto end;
7278                         }
7279
7280                         offset[i] = mf->offset;
7281                 }
7282
7283         /* Check that there are no duplicated match fields. */
7284         for (i = 0; i < params->n_fields; i++) {
7285                 uint32_t j;
7286
7287                 for (j = 0; j < i; j++)
7288                         if (offset[j] == offset[i]) {
7289                                 status = -EINVAL;
7290                                 goto end;
7291                         }
7292         }
7293
7294         /* Return. */
7295         if (header)
7296                 *header = h0;
7297
7298 end:
7299         free(offset);
7300         return status;
7301 }
7302
7303 int
7304 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7305                               const char *name,
7306                               struct rte_swx_pipeline_table_params *params,
7307                               const char *recommended_table_type_name,
7308                               const char *args,
7309                               uint32_t size)
7310 {
7311         struct table_type *type;
7312         struct table *t = NULL;
7313         struct action *default_action;
7314         struct header *header = NULL;
7315         uint32_t action_data_size_max = 0, i;
7316         int status = 0;
7317
7318         CHECK(p, EINVAL);
7319
7320         CHECK_NAME(name, EINVAL);
7321         CHECK(!table_find(p, name), EEXIST);
7322         CHECK(!selector_find(p, name), EEXIST);
7323         CHECK(!learner_find(p, name), EEXIST);
7324
7325         CHECK(params, EINVAL);
7326
7327         /* Match checks. */
7328         status = table_match_fields_check(p, params, &header);
7329         if (status)
7330                 return status;
7331
7332         /* Action checks. */
7333         CHECK(params->n_actions, EINVAL);
7334         CHECK(params->action_names, EINVAL);
7335         for (i = 0; i < params->n_actions; i++) {
7336                 const char *action_name = params->action_names[i];
7337                 struct action *a;
7338                 uint32_t action_data_size;
7339                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7340
7341                 CHECK_NAME(action_name, EINVAL);
7342
7343                 a = action_find(p, action_name);
7344                 CHECK(a, EINVAL);
7345                 CHECK(!action_does_learning(a), EINVAL);
7346
7347                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7348                 if (action_data_size > action_data_size_max)
7349                         action_data_size_max = action_data_size;
7350
7351                 if (params->action_is_for_table_entries)
7352                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7353                 if (params->action_is_for_default_entry)
7354                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7355                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7356         }
7357
7358         CHECK_NAME(params->default_action_name, EINVAL);
7359         for (i = 0; i < p->n_actions; i++)
7360                 if (!strcmp(params->action_names[i],
7361                             params->default_action_name))
7362                         break;
7363         CHECK(i < params->n_actions, EINVAL);
7364         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7365               EINVAL);
7366
7367         default_action = action_find(p, params->default_action_name);
7368         CHECK((default_action->st && params->default_action_data) ||
7369               !params->default_action_data, EINVAL);
7370
7371         /* Table type checks. */
7372         if (recommended_table_type_name)
7373                 CHECK_NAME(recommended_table_type_name, EINVAL);
7374
7375         if (params->n_fields) {
7376                 enum rte_swx_table_match_type match_type;
7377
7378                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7379                 if (status)
7380                         return status;
7381
7382                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7383                 CHECK(type, EINVAL);
7384         } else {
7385                 type = NULL;
7386         }
7387
7388         /* Memory allocation. */
7389         t = calloc(1, sizeof(struct table));
7390         if (!t)
7391                 goto nomem;
7392
7393         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7394         if (!t->fields)
7395                 goto nomem;
7396
7397         t->actions = calloc(params->n_actions, sizeof(struct action *));
7398         if (!t->actions)
7399                 goto nomem;
7400
7401         if (action_data_size_max) {
7402                 t->default_action_data = calloc(1, action_data_size_max);
7403                 if (!t->default_action_data)
7404                         goto nomem;
7405         }
7406
7407         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7408         if (!t->action_is_for_table_entries)
7409                 goto nomem;
7410
7411         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7412         if (!t->action_is_for_default_entry)
7413                 goto nomem;
7414
7415         /* Node initialization. */
7416         strcpy(t->name, name);
7417         if (args && args[0])
7418                 strcpy(t->args, args);
7419         t->type = type;
7420
7421         for (i = 0; i < params->n_fields; i++) {
7422                 struct rte_swx_match_field_params *field = &params->fields[i];
7423                 struct match_field *f = &t->fields[i];
7424
7425                 f->match_type = field->match_type;
7426                 f->field = header ?
7427                         header_field_parse(p, field->name, NULL) :
7428                         metadata_field_parse(p, field->name);
7429         }
7430         t->n_fields = params->n_fields;
7431         t->header = header;
7432
7433         for (i = 0; i < params->n_actions; i++) {
7434                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7435
7436                 if (params->action_is_for_table_entries)
7437                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7438                 if (params->action_is_for_default_entry)
7439                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7440
7441                 t->actions[i] = action_find(p, params->action_names[i]);
7442                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7443                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7444         }
7445         t->default_action = default_action;
7446         if (default_action->st)
7447                 memcpy(t->default_action_data,
7448                        params->default_action_data,
7449                        default_action->st->n_bits / 8);
7450         t->n_actions = params->n_actions;
7451         t->default_action_is_const = params->default_action_is_const;
7452         t->action_data_size_max = action_data_size_max;
7453
7454         t->size = size;
7455         t->id = p->n_tables;
7456
7457         /* Node add to tailq. */
7458         TAILQ_INSERT_TAIL(&p->tables, t, node);
7459         p->n_tables++;
7460
7461         return 0;
7462
7463 nomem:
7464         if (!t)
7465                 return -ENOMEM;
7466
7467         free(t->action_is_for_default_entry);
7468         free(t->action_is_for_table_entries);
7469         free(t->default_action_data);
7470         free(t->actions);
7471         free(t->fields);
7472         free(t);
7473
7474         return -ENOMEM;
7475 }
7476
7477 static struct rte_swx_table_params *
7478 table_params_get(struct table *table)
7479 {
7480         struct rte_swx_table_params *params;
7481         struct field *first, *last;
7482         uint8_t *key_mask;
7483         uint32_t key_size, key_offset, action_data_size, i;
7484
7485         /* Memory allocation. */
7486         params = calloc(1, sizeof(struct rte_swx_table_params));
7487         if (!params)
7488                 return NULL;
7489
7490         /* Find first (smallest offset) and last (biggest offset) match fields. */
7491         first = table->fields[0].field;
7492         last = table->fields[0].field;
7493
7494         for (i = 0; i < table->n_fields; i++) {
7495                 struct field *f = table->fields[i].field;
7496
7497                 if (f->offset < first->offset)
7498                         first = f;
7499
7500                 if (f->offset > last->offset)
7501                         last = f;
7502         }
7503
7504         /* Key offset and size. */
7505         key_offset = first->offset / 8;
7506         key_size = (last->offset + last->n_bits - first->offset) / 8;
7507
7508         /* Memory allocation. */
7509         key_mask = calloc(1, key_size);
7510         if (!key_mask) {
7511                 free(params);
7512                 return NULL;
7513         }
7514
7515         /* Key mask. */
7516         for (i = 0; i < table->n_fields; i++) {
7517                 struct field *f = table->fields[i].field;
7518                 uint32_t start = (f->offset - first->offset) / 8;
7519                 size_t size = f->n_bits / 8;
7520
7521                 memset(&key_mask[start], 0xFF, size);
7522         }
7523
7524         /* Action data size. */
7525         action_data_size = 0;
7526         for (i = 0; i < table->n_actions; i++) {
7527                 struct action *action = table->actions[i];
7528                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7529
7530                 if (ads > action_data_size)
7531                         action_data_size = ads;
7532         }
7533
7534         /* Fill in. */
7535         params->match_type = table->type->match_type;
7536         params->key_size = key_size;
7537         params->key_offset = key_offset;
7538         params->key_mask0 = key_mask;
7539         params->action_data_size = action_data_size;
7540         params->n_keys_max = table->size;
7541
7542         return params;
7543 }
7544
7545 static void
7546 table_params_free(struct rte_swx_table_params *params)
7547 {
7548         if (!params)
7549                 return;
7550
7551         free(params->key_mask0);
7552         free(params);
7553 }
7554
7555 static int
7556 table_stub_lkp(void *table __rte_unused,
7557                void *mailbox __rte_unused,
7558                uint8_t **key __rte_unused,
7559                uint64_t *action_id __rte_unused,
7560                uint8_t **action_data __rte_unused,
7561                int *hit)
7562 {
7563         *hit = 0;
7564         return 1; /* DONE. */
7565 }
7566
7567 static int
7568 table_build(struct rte_swx_pipeline *p)
7569 {
7570         uint32_t i;
7571
7572         /* Per pipeline: table statistics. */
7573         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7574         CHECK(p->table_stats, ENOMEM);
7575
7576         for (i = 0; i < p->n_tables; i++) {
7577                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7578                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7579         }
7580
7581         /* Per thread: table runt-time. */
7582         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7583                 struct thread *t = &p->threads[i];
7584                 struct table *table;
7585
7586                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7587                 CHECK(t->tables, ENOMEM);
7588
7589                 TAILQ_FOREACH(table, &p->tables, node) {
7590                         struct table_runtime *r = &t->tables[table->id];
7591
7592                         if (table->type) {
7593                                 uint64_t size;
7594
7595                                 size = table->type->ops.mailbox_size_get();
7596
7597                                 /* r->func. */
7598                                 r->func = table->type->ops.lkp;
7599
7600                                 /* r->mailbox. */
7601                                 if (size) {
7602                                         r->mailbox = calloc(1, size);
7603                                         CHECK(r->mailbox, ENOMEM);
7604                                 }
7605
7606                                 /* r->key. */
7607                                 r->key = table->header ?
7608                                         &t->structs[table->header->struct_id] :
7609                                         &t->structs[p->metadata_struct_id];
7610                         } else {
7611                                 r->func = table_stub_lkp;
7612                         }
7613                 }
7614         }
7615
7616         return 0;
7617 }
7618
7619 static void
7620 table_build_free(struct rte_swx_pipeline *p)
7621 {
7622         uint32_t i;
7623
7624         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7625                 struct thread *t = &p->threads[i];
7626                 uint32_t j;
7627
7628                 if (!t->tables)
7629                         continue;
7630
7631                 for (j = 0; j < p->n_tables; j++) {
7632                         struct table_runtime *r = &t->tables[j];
7633
7634                         free(r->mailbox);
7635                 }
7636
7637                 free(t->tables);
7638                 t->tables = NULL;
7639         }
7640
7641         if (p->table_stats) {
7642                 for (i = 0; i < p->n_tables; i++)
7643                         free(p->table_stats[i].n_pkts_action);
7644
7645                 free(p->table_stats);
7646         }
7647 }
7648
7649 static void
7650 table_free(struct rte_swx_pipeline *p)
7651 {
7652         table_build_free(p);
7653
7654         /* Tables. */
7655         for ( ; ; ) {
7656                 struct table *elem;
7657
7658                 elem = TAILQ_FIRST(&p->tables);
7659                 if (!elem)
7660                         break;
7661
7662                 TAILQ_REMOVE(&p->tables, elem, node);
7663                 free(elem->fields);
7664                 free(elem->actions);
7665                 free(elem->default_action_data);
7666                 free(elem);
7667         }
7668
7669         /* Table types. */
7670         for ( ; ; ) {
7671                 struct table_type *elem;
7672
7673                 elem = TAILQ_FIRST(&p->table_types);
7674                 if (!elem)
7675                         break;
7676
7677                 TAILQ_REMOVE(&p->table_types, elem, node);
7678                 free(elem);
7679         }
7680 }
7681
7682 /*
7683  * Selector.
7684  */
7685 static struct selector *
7686 selector_find(struct rte_swx_pipeline *p, const char *name)
7687 {
7688         struct selector *s;
7689
7690         TAILQ_FOREACH(s, &p->selectors, node)
7691                 if (strcmp(s->name, name) == 0)
7692                         return s;
7693
7694         return NULL;
7695 }
7696
7697 static struct selector *
7698 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7699 {
7700         struct selector *s = NULL;
7701
7702         TAILQ_FOREACH(s, &p->selectors, node)
7703                 if (s->id == id)
7704                         return s;
7705
7706         return NULL;
7707 }
7708
7709 static int
7710 selector_fields_check(struct rte_swx_pipeline *p,
7711                       struct rte_swx_pipeline_selector_params *params,
7712                       struct header **header)
7713 {
7714         struct header *h0 = NULL;
7715         struct field *hf, *mf;
7716         uint32_t i;
7717
7718         /* Return if no selector fields. */
7719         if (!params->n_selector_fields || !params->selector_field_names)
7720                 return -EINVAL;
7721
7722         /* Check that all the selector fields either belong to the same header
7723          * or are all meta-data fields.
7724          */
7725         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7726         mf = metadata_field_parse(p, params->selector_field_names[0]);
7727         if (!hf && !mf)
7728                 return -EINVAL;
7729
7730         for (i = 1; i < params->n_selector_fields; i++)
7731                 if (h0) {
7732                         struct header *h;
7733
7734                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7735                         if (!hf || (h->id != h0->id))
7736                                 return -EINVAL;
7737                 } else {
7738                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7739                         if (!mf)
7740                                 return -EINVAL;
7741                 }
7742
7743         /* Check that there are no duplicated match fields. */
7744         for (i = 0; i < params->n_selector_fields; i++) {
7745                 const char *field_name = params->selector_field_names[i];
7746                 uint32_t j;
7747
7748                 for (j = i + 1; j < params->n_selector_fields; j++)
7749                         if (!strcmp(params->selector_field_names[j], field_name))
7750                                 return -EINVAL;
7751         }
7752
7753         /* Return. */
7754         if (header)
7755                 *header = h0;
7756
7757         return 0;
7758 }
7759
7760 int
7761 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7762                                  const char *name,
7763                                  struct rte_swx_pipeline_selector_params *params)
7764 {
7765         struct selector *s;
7766         struct header *selector_header = NULL;
7767         struct field *group_id_field, *member_id_field;
7768         uint32_t i;
7769         int status = 0;
7770
7771         CHECK(p, EINVAL);
7772
7773         CHECK_NAME(name, EINVAL);
7774         CHECK(!table_find(p, name), EEXIST);
7775         CHECK(!selector_find(p, name), EEXIST);
7776         CHECK(!learner_find(p, name), EEXIST);
7777
7778         CHECK(params, EINVAL);
7779
7780         CHECK_NAME(params->group_id_field_name, EINVAL);
7781         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7782         CHECK(group_id_field, EINVAL);
7783
7784         for (i = 0; i < params->n_selector_fields; i++) {
7785                 const char *field_name = params->selector_field_names[i];
7786
7787                 CHECK_NAME(field_name, EINVAL);
7788         }
7789         status = selector_fields_check(p, params, &selector_header);
7790         if (status)
7791                 return status;
7792
7793         CHECK_NAME(params->member_id_field_name, EINVAL);
7794         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7795         CHECK(member_id_field, EINVAL);
7796
7797         CHECK(params->n_groups_max, EINVAL);
7798
7799         CHECK(params->n_members_per_group_max, EINVAL);
7800
7801         /* Memory allocation. */
7802         s = calloc(1, sizeof(struct selector));
7803         if (!s) {
7804                 status = -ENOMEM;
7805                 goto error;
7806         }
7807
7808         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7809         if (!s->selector_fields) {
7810                 status = -ENOMEM;
7811                 goto error;
7812         }
7813
7814         /* Node initialization. */
7815         strcpy(s->name, name);
7816
7817         s->group_id_field = group_id_field;
7818
7819         for (i = 0; i < params->n_selector_fields; i++) {
7820                 const char *field_name = params->selector_field_names[i];
7821
7822                 s->selector_fields[i] = selector_header ?
7823                         header_field_parse(p, field_name, NULL) :
7824                         metadata_field_parse(p, field_name);
7825         }
7826
7827         s->n_selector_fields = params->n_selector_fields;
7828
7829         s->selector_header = selector_header;
7830
7831         s->member_id_field = member_id_field;
7832
7833         s->n_groups_max = params->n_groups_max;
7834
7835         s->n_members_per_group_max = params->n_members_per_group_max;
7836
7837         s->id = p->n_selectors;
7838
7839         /* Node add to tailq. */
7840         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7841         p->n_selectors++;
7842
7843         return 0;
7844
7845 error:
7846         if (!s)
7847                 return status;
7848
7849         free(s->selector_fields);
7850
7851         free(s);
7852
7853         return status;
7854 }
7855
7856 static void
7857 selector_params_free(struct rte_swx_table_selector_params *params)
7858 {
7859         if (!params)
7860                 return;
7861
7862         free(params->selector_mask);
7863
7864         free(params);
7865 }
7866
7867 static struct rte_swx_table_selector_params *
7868 selector_table_params_get(struct selector *s)
7869 {
7870         struct rte_swx_table_selector_params *params = NULL;
7871         struct field *first, *last;
7872         uint32_t i;
7873
7874         /* Memory allocation. */
7875         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7876         if (!params)
7877                 goto error;
7878
7879         /* Group ID. */
7880         params->group_id_offset = s->group_id_field->offset / 8;
7881
7882         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7883         first = s->selector_fields[0];
7884         last = s->selector_fields[0];
7885
7886         for (i = 0; i < s->n_selector_fields; i++) {
7887                 struct field *f = s->selector_fields[i];
7888
7889                 if (f->offset < first->offset)
7890                         first = f;
7891
7892                 if (f->offset > last->offset)
7893                         last = f;
7894         }
7895
7896         /* Selector offset and size. */
7897         params->selector_offset = first->offset / 8;
7898         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7899
7900         /* Memory allocation. */
7901         params->selector_mask = calloc(1, params->selector_size);
7902         if (!params->selector_mask)
7903                 goto error;
7904
7905         /* Selector mask. */
7906         for (i = 0; i < s->n_selector_fields; i++) {
7907                 struct field *f = s->selector_fields[i];
7908                 uint32_t start = (f->offset - first->offset) / 8;
7909                 size_t size = f->n_bits / 8;
7910
7911                 memset(&params->selector_mask[start], 0xFF, size);
7912         }
7913
7914         /* Member ID. */
7915         params->member_id_offset = s->member_id_field->offset / 8;
7916
7917         /* Maximum number of groups. */
7918         params->n_groups_max = s->n_groups_max;
7919
7920         /* Maximum number of members per group. */
7921         params->n_members_per_group_max = s->n_members_per_group_max;
7922
7923         return params;
7924
7925 error:
7926         selector_params_free(params);
7927         return NULL;
7928 }
7929
7930 static void
7931 selector_build_free(struct rte_swx_pipeline *p)
7932 {
7933         uint32_t i;
7934
7935         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7936                 struct thread *t = &p->threads[i];
7937                 uint32_t j;
7938
7939                 if (!t->selectors)
7940                         continue;
7941
7942                 for (j = 0; j < p->n_selectors; j++) {
7943                         struct selector_runtime *r = &t->selectors[j];
7944
7945                         free(r->mailbox);
7946                 }
7947
7948                 free(t->selectors);
7949                 t->selectors = NULL;
7950         }
7951
7952         free(p->selector_stats);
7953         p->selector_stats = NULL;
7954 }
7955
7956 static int
7957 selector_build(struct rte_swx_pipeline *p)
7958 {
7959         uint32_t i;
7960         int status = 0;
7961
7962         /* Per pipeline: selector statistics. */
7963         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7964         if (!p->selector_stats) {
7965                 status = -ENOMEM;
7966                 goto error;
7967         }
7968
7969         /* Per thread: selector run-time. */
7970         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7971                 struct thread *t = &p->threads[i];
7972                 struct selector *s;
7973
7974                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7975                 if (!t->selectors) {
7976                         status = -ENOMEM;
7977                         goto error;
7978                 }
7979
7980                 TAILQ_FOREACH(s, &p->selectors, node) {
7981                         struct selector_runtime *r = &t->selectors[s->id];
7982                         uint64_t size;
7983
7984                         /* r->mailbox. */
7985                         size = rte_swx_table_selector_mailbox_size_get();
7986                         if (size) {
7987                                 r->mailbox = calloc(1, size);
7988                                 if (!r->mailbox) {
7989                                         status = -ENOMEM;
7990                                         goto error;
7991                                 }
7992                         }
7993
7994                         /* r->group_id_buffer. */
7995                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
7996
7997                         /* r->selector_buffer. */
7998                         r->selector_buffer = s->selector_header ?
7999                                 &t->structs[s->selector_header->struct_id] :
8000                                 &t->structs[p->metadata_struct_id];
8001
8002                         /* r->member_id_buffer. */
8003                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8004                 }
8005         }
8006
8007         return 0;
8008
8009 error:
8010         selector_build_free(p);
8011         return status;
8012 }
8013
8014 static void
8015 selector_free(struct rte_swx_pipeline *p)
8016 {
8017         selector_build_free(p);
8018
8019         /* Selector tables. */
8020         for ( ; ; ) {
8021                 struct selector *elem;
8022
8023                 elem = TAILQ_FIRST(&p->selectors);
8024                 if (!elem)
8025                         break;
8026
8027                 TAILQ_REMOVE(&p->selectors, elem, node);
8028                 free(elem->selector_fields);
8029                 free(elem);
8030         }
8031 }
8032
8033 /*
8034  * Learner table.
8035  */
8036 static struct learner *
8037 learner_find(struct rte_swx_pipeline *p, const char *name)
8038 {
8039         struct learner *l;
8040
8041         TAILQ_FOREACH(l, &p->learners, node)
8042                 if (!strcmp(l->name, name))
8043                         return l;
8044
8045         return NULL;
8046 }
8047
8048 static struct learner *
8049 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8050 {
8051         struct learner *l = NULL;
8052
8053         TAILQ_FOREACH(l, &p->learners, node)
8054                 if (l->id == id)
8055                         return l;
8056
8057         return NULL;
8058 }
8059
8060 static int
8061 learner_match_fields_check(struct rte_swx_pipeline *p,
8062                            struct rte_swx_pipeline_learner_params *params,
8063                            struct header **header)
8064 {
8065         struct header *h0 = NULL;
8066         struct field *hf, *mf;
8067         uint32_t i;
8068
8069         /* Return if no match fields. */
8070         if (!params->n_fields || !params->field_names)
8071                 return -EINVAL;
8072
8073         /* Check that all the match fields either belong to the same header
8074          * or are all meta-data fields.
8075          */
8076         hf = header_field_parse(p, params->field_names[0], &h0);
8077         mf = metadata_field_parse(p, params->field_names[0]);
8078         if (!hf && !mf)
8079                 return -EINVAL;
8080
8081         for (i = 1; i < params->n_fields; i++)
8082                 if (h0) {
8083                         struct header *h;
8084
8085                         hf = header_field_parse(p, params->field_names[i], &h);
8086                         if (!hf || (h->id != h0->id))
8087                                 return -EINVAL;
8088                 } else {
8089                         mf = metadata_field_parse(p, params->field_names[i]);
8090                         if (!mf)
8091                                 return -EINVAL;
8092                 }
8093
8094         /* Check that there are no duplicated match fields. */
8095         for (i = 0; i < params->n_fields; i++) {
8096                 const char *field_name = params->field_names[i];
8097                 uint32_t j;
8098
8099                 for (j = i + 1; j < params->n_fields; j++)
8100                         if (!strcmp(params->field_names[j], field_name))
8101                                 return -EINVAL;
8102         }
8103
8104         /* Return. */
8105         if (header)
8106                 *header = h0;
8107
8108         return 0;
8109 }
8110
8111 static int
8112 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8113 {
8114         struct struct_type *mst = p->metadata_st, *ast = a->st;
8115         struct field *mf, *af;
8116         uint32_t mf_pos, i;
8117
8118         if (!ast) {
8119                 if (mf_name)
8120                         return -EINVAL;
8121
8122                 return 0;
8123         }
8124
8125         /* Check that mf_name is the name of a valid meta-data field. */
8126         CHECK_NAME(mf_name, EINVAL);
8127         mf = metadata_field_parse(p, mf_name);
8128         CHECK(mf, EINVAL);
8129
8130         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8131          * all the action arguments.
8132          */
8133         mf_pos = mf - mst->fields;
8134         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8135
8136         /* Check that the size of each of the identified meta-data fields matches exactly the size
8137          * of the corresponding action argument.
8138          */
8139         for (i = 0; i < ast->n_fields; i++) {
8140                 mf = &mst->fields[mf_pos + i];
8141                 af = &ast->fields[i];
8142
8143                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8144         }
8145
8146         return 0;
8147 }
8148
8149 static int
8150 learner_action_learning_check(struct rte_swx_pipeline *p,
8151                               struct action *action,
8152                               const char **action_names,
8153                               uint32_t n_actions)
8154 {
8155         uint32_t i;
8156
8157         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8158          * the action passed as argument to the "learn" instruction) is also enabled for the
8159          * current learner table.
8160          */
8161         for (i = 0; i < action->n_instructions; i++) {
8162                 struct instruction *instr = &action->instructions[i];
8163                 uint32_t found = 0, j;
8164
8165                 if (instr->type != INSTR_LEARNER_LEARN)
8166                         continue;
8167
8168                 for (j = 0; j < n_actions; j++) {
8169                         struct action *a;
8170
8171                         a = action_find(p, action_names[j]);
8172                         if (!a)
8173                                 return -EINVAL;
8174
8175                         if (a->id == instr->learn.action_id)
8176                                 found = 1;
8177                 }
8178
8179                 if (!found)
8180                         return -EINVAL;
8181         }
8182
8183         return 0;
8184 }
8185
8186 int
8187 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8188                               const char *name,
8189                               struct rte_swx_pipeline_learner_params *params,
8190                               uint32_t size,
8191                               uint32_t timeout)
8192 {
8193         struct learner *l = NULL;
8194         struct action *default_action;
8195         struct header *header = NULL;
8196         uint32_t action_data_size_max = 0, i;
8197         int status = 0;
8198
8199         CHECK(p, EINVAL);
8200
8201         CHECK_NAME(name, EINVAL);
8202         CHECK(!table_find(p, name), EEXIST);
8203         CHECK(!selector_find(p, name), EEXIST);
8204         CHECK(!learner_find(p, name), EEXIST);
8205
8206         CHECK(params, EINVAL);
8207
8208         /* Match checks. */
8209         status = learner_match_fields_check(p, params, &header);
8210         if (status)
8211                 return status;
8212
8213         /* Action checks. */
8214         CHECK(params->n_actions, EINVAL);
8215         CHECK(params->action_names, EINVAL);
8216         for (i = 0; i < params->n_actions; i++) {
8217                 const char *action_name = params->action_names[i];
8218                 struct action *a;
8219                 uint32_t action_data_size;
8220                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8221
8222                 CHECK_NAME(action_name, EINVAL);
8223
8224                 a = action_find(p, action_name);
8225                 CHECK(a, EINVAL);
8226
8227                 status = learner_action_learning_check(p,
8228                                                        a,
8229                                                        params->action_names,
8230                                                        params->n_actions);
8231                 if (status)
8232                         return status;
8233
8234                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8235                 if (action_data_size > action_data_size_max)
8236                         action_data_size_max = action_data_size;
8237
8238                 if (params->action_is_for_table_entries)
8239                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8240                 if (params->action_is_for_default_entry)
8241                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8242                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8243         }
8244
8245         CHECK_NAME(params->default_action_name, EINVAL);
8246         for (i = 0; i < p->n_actions; i++)
8247                 if (!strcmp(params->action_names[i],
8248                             params->default_action_name))
8249                         break;
8250         CHECK(i < params->n_actions, EINVAL);
8251         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8252               EINVAL);
8253
8254         default_action = action_find(p, params->default_action_name);
8255         CHECK((default_action->st && params->default_action_data) ||
8256               !params->default_action_data, EINVAL);
8257
8258         /* Any other checks. */
8259         CHECK(size, EINVAL);
8260         CHECK(timeout, EINVAL);
8261
8262         /* Memory allocation. */
8263         l = calloc(1, sizeof(struct learner));
8264         if (!l)
8265                 goto nomem;
8266
8267         l->fields = calloc(params->n_fields, sizeof(struct field *));
8268         if (!l->fields)
8269                 goto nomem;
8270
8271         l->actions = calloc(params->n_actions, sizeof(struct action *));
8272         if (!l->actions)
8273                 goto nomem;
8274
8275         if (action_data_size_max) {
8276                 l->default_action_data = calloc(1, action_data_size_max);
8277                 if (!l->default_action_data)
8278                         goto nomem;
8279         }
8280
8281         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8282         if (!l->action_is_for_table_entries)
8283                 goto nomem;
8284
8285         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8286         if (!l->action_is_for_default_entry)
8287                 goto nomem;
8288
8289         /* Node initialization. */
8290         strcpy(l->name, name);
8291
8292         for (i = 0; i < params->n_fields; i++) {
8293                 const char *field_name = params->field_names[i];
8294
8295                 l->fields[i] = header ?
8296                         header_field_parse(p, field_name, NULL) :
8297                         metadata_field_parse(p, field_name);
8298         }
8299
8300         l->n_fields = params->n_fields;
8301
8302         l->header = header;
8303
8304         for (i = 0; i < params->n_actions; i++) {
8305                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8306
8307                 if (params->action_is_for_table_entries)
8308                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8309                 if (params->action_is_for_default_entry)
8310                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8311
8312                 l->actions[i] = action_find(p, params->action_names[i]);
8313                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8314                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8315         }
8316
8317         l->default_action = default_action;
8318
8319         if (default_action->st)
8320                 memcpy(l->default_action_data,
8321                        params->default_action_data,
8322                        default_action->st->n_bits / 8);
8323
8324         l->n_actions = params->n_actions;
8325
8326         l->default_action_is_const = params->default_action_is_const;
8327
8328         l->action_data_size_max = action_data_size_max;
8329
8330         l->size = size;
8331
8332         l->timeout = timeout;
8333
8334         l->id = p->n_learners;
8335
8336         /* Node add to tailq. */
8337         TAILQ_INSERT_TAIL(&p->learners, l, node);
8338         p->n_learners++;
8339
8340         return 0;
8341
8342 nomem:
8343         if (!l)
8344                 return -ENOMEM;
8345
8346         free(l->action_is_for_default_entry);
8347         free(l->action_is_for_table_entries);
8348         free(l->default_action_data);
8349         free(l->actions);
8350         free(l->fields);
8351         free(l);
8352
8353         return -ENOMEM;
8354 }
8355
8356 static void
8357 learner_params_free(struct rte_swx_table_learner_params *params)
8358 {
8359         if (!params)
8360                 return;
8361
8362         free(params->key_mask0);
8363
8364         free(params);
8365 }
8366
8367 static struct rte_swx_table_learner_params *
8368 learner_params_get(struct learner *l)
8369 {
8370         struct rte_swx_table_learner_params *params = NULL;
8371         struct field *first, *last;
8372         uint32_t i;
8373
8374         /* Memory allocation. */
8375         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8376         if (!params)
8377                 goto error;
8378
8379         /* Find first (smallest offset) and last (biggest offset) match fields. */
8380         first = l->fields[0];
8381         last = l->fields[0];
8382
8383         for (i = 0; i < l->n_fields; i++) {
8384                 struct field *f = l->fields[i];
8385
8386                 if (f->offset < first->offset)
8387                         first = f;
8388
8389                 if (f->offset > last->offset)
8390                         last = f;
8391         }
8392
8393         /* Key offset and size. */
8394         params->key_offset = first->offset / 8;
8395         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8396
8397         /* Memory allocation. */
8398         params->key_mask0 = calloc(1, params->key_size);
8399         if (!params->key_mask0)
8400                 goto error;
8401
8402         /* Key mask. */
8403         for (i = 0; i < l->n_fields; i++) {
8404                 struct field *f = l->fields[i];
8405                 uint32_t start = (f->offset - first->offset) / 8;
8406                 size_t size = f->n_bits / 8;
8407
8408                 memset(&params->key_mask0[start], 0xFF, size);
8409         }
8410
8411         /* Action data size. */
8412         params->action_data_size = l->action_data_size_max;
8413
8414         /* Maximum number of keys. */
8415         params->n_keys_max = l->size;
8416
8417         /* Timeout. */
8418         params->key_timeout = l->timeout;
8419
8420         return params;
8421
8422 error:
8423         learner_params_free(params);
8424         return NULL;
8425 }
8426
8427 static void
8428 learner_build_free(struct rte_swx_pipeline *p)
8429 {
8430         uint32_t i;
8431
8432         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8433                 struct thread *t = &p->threads[i];
8434                 uint32_t j;
8435
8436                 if (!t->learners)
8437                         continue;
8438
8439                 for (j = 0; j < p->n_learners; j++) {
8440                         struct learner_runtime *r = &t->learners[j];
8441
8442                         free(r->mailbox);
8443                 }
8444
8445                 free(t->learners);
8446                 t->learners = NULL;
8447         }
8448
8449         if (p->learner_stats) {
8450                 for (i = 0; i < p->n_learners; i++)
8451                         free(p->learner_stats[i].n_pkts_action);
8452
8453                 free(p->learner_stats);
8454         }
8455 }
8456
8457 static int
8458 learner_build(struct rte_swx_pipeline *p)
8459 {
8460         uint32_t i;
8461         int status = 0;
8462
8463         /* Per pipeline: learner statistics. */
8464         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8465         CHECK(p->learner_stats, ENOMEM);
8466
8467         for (i = 0; i < p->n_learners; i++) {
8468                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8469                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8470         }
8471
8472         /* Per thread: learner run-time. */
8473         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8474                 struct thread *t = &p->threads[i];
8475                 struct learner *l;
8476
8477                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8478                 if (!t->learners) {
8479                         status = -ENOMEM;
8480                         goto error;
8481                 }
8482
8483                 TAILQ_FOREACH(l, &p->learners, node) {
8484                         struct learner_runtime *r = &t->learners[l->id];
8485                         uint64_t size;
8486
8487                         /* r->mailbox. */
8488                         size = rte_swx_table_learner_mailbox_size_get();
8489                         if (size) {
8490                                 r->mailbox = calloc(1, size);
8491                                 if (!r->mailbox) {
8492                                         status = -ENOMEM;
8493                                         goto error;
8494                                 }
8495                         }
8496
8497                         /* r->key. */
8498                         r->key = l->header ?
8499                                 &t->structs[l->header->struct_id] :
8500                                 &t->structs[p->metadata_struct_id];
8501                 }
8502         }
8503
8504         return 0;
8505
8506 error:
8507         learner_build_free(p);
8508         return status;
8509 }
8510
8511 static void
8512 learner_free(struct rte_swx_pipeline *p)
8513 {
8514         learner_build_free(p);
8515
8516         /* Learner tables. */
8517         for ( ; ; ) {
8518                 struct learner *l;
8519
8520                 l = TAILQ_FIRST(&p->learners);
8521                 if (!l)
8522                         break;
8523
8524                 TAILQ_REMOVE(&p->learners, l, node);
8525                 free(l->fields);
8526                 free(l->actions);
8527                 free(l->default_action_data);
8528                 free(l);
8529         }
8530 }
8531
8532 /*
8533  * Table state.
8534  */
8535 static int
8536 table_state_build(struct rte_swx_pipeline *p)
8537 {
8538         struct table *table;
8539         struct selector *s;
8540         struct learner *l;
8541
8542         p->table_state = calloc(p->n_tables + p->n_selectors,
8543                                 sizeof(struct rte_swx_table_state));
8544         CHECK(p->table_state, ENOMEM);
8545
8546         TAILQ_FOREACH(table, &p->tables, node) {
8547                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8548
8549                 if (table->type) {
8550                         struct rte_swx_table_params *params;
8551
8552                         /* ts->obj. */
8553                         params = table_params_get(table);
8554                         CHECK(params, ENOMEM);
8555
8556                         ts->obj = table->type->ops.create(params,
8557                                 NULL,
8558                                 table->args,
8559                                 p->numa_node);
8560
8561                         table_params_free(params);
8562                         CHECK(ts->obj, ENODEV);
8563                 }
8564
8565                 /* ts->default_action_data. */
8566                 if (table->action_data_size_max) {
8567                         ts->default_action_data =
8568                                 malloc(table->action_data_size_max);
8569                         CHECK(ts->default_action_data, ENOMEM);
8570
8571                         memcpy(ts->default_action_data,
8572                                table->default_action_data,
8573                                table->action_data_size_max);
8574                 }
8575
8576                 /* ts->default_action_id. */
8577                 ts->default_action_id = table->default_action->id;
8578         }
8579
8580         TAILQ_FOREACH(s, &p->selectors, node) {
8581                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8582                 struct rte_swx_table_selector_params *params;
8583
8584                 /* ts->obj. */
8585                 params = selector_table_params_get(s);
8586                 CHECK(params, ENOMEM);
8587
8588                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8589
8590                 selector_params_free(params);
8591                 CHECK(ts->obj, ENODEV);
8592         }
8593
8594         TAILQ_FOREACH(l, &p->learners, node) {
8595                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8596                         p->n_selectors + l->id];
8597                 struct rte_swx_table_learner_params *params;
8598
8599                 /* ts->obj. */
8600                 params = learner_params_get(l);
8601                 CHECK(params, ENOMEM);
8602
8603                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8604                 learner_params_free(params);
8605                 CHECK(ts->obj, ENODEV);
8606
8607                 /* ts->default_action_data. */
8608                 if (l->action_data_size_max) {
8609                         ts->default_action_data = malloc(l->action_data_size_max);
8610                         CHECK(ts->default_action_data, ENOMEM);
8611
8612                         memcpy(ts->default_action_data,
8613                                l->default_action_data,
8614                                l->action_data_size_max);
8615                 }
8616
8617                 /* ts->default_action_id. */
8618                 ts->default_action_id = l->default_action->id;
8619         }
8620
8621         return 0;
8622 }
8623
8624 static void
8625 table_state_build_free(struct rte_swx_pipeline *p)
8626 {
8627         uint32_t i;
8628
8629         if (!p->table_state)
8630                 return;
8631
8632         for (i = 0; i < p->n_tables; i++) {
8633                 struct rte_swx_table_state *ts = &p->table_state[i];
8634                 struct table *table = table_find_by_id(p, i);
8635
8636                 /* ts->obj. */
8637                 if (table->type && ts->obj)
8638                         table->type->ops.free(ts->obj);
8639
8640                 /* ts->default_action_data. */
8641                 free(ts->default_action_data);
8642         }
8643
8644         for (i = 0; i < p->n_selectors; i++) {
8645                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8646
8647                 /* ts->obj. */
8648                 if (ts->obj)
8649                         rte_swx_table_selector_free(ts->obj);
8650         }
8651
8652         for (i = 0; i < p->n_learners; i++) {
8653                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8654
8655                 /* ts->obj. */
8656                 if (ts->obj)
8657                         rte_swx_table_learner_free(ts->obj);
8658
8659                 /* ts->default_action_data. */
8660                 free(ts->default_action_data);
8661         }
8662
8663         free(p->table_state);
8664         p->table_state = NULL;
8665 }
8666
8667 static void
8668 table_state_free(struct rte_swx_pipeline *p)
8669 {
8670         table_state_build_free(p);
8671 }
8672
8673 /*
8674  * Register array.
8675  */
8676 static struct regarray *
8677 regarray_find(struct rte_swx_pipeline *p, const char *name)
8678 {
8679         struct regarray *elem;
8680
8681         TAILQ_FOREACH(elem, &p->regarrays, node)
8682                 if (!strcmp(elem->name, name))
8683                         return elem;
8684
8685         return NULL;
8686 }
8687
8688 static struct regarray *
8689 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8690 {
8691         struct regarray *elem = NULL;
8692
8693         TAILQ_FOREACH(elem, &p->regarrays, node)
8694                 if (elem->id == id)
8695                         return elem;
8696
8697         return NULL;
8698 }
8699
8700 int
8701 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8702                               const char *name,
8703                               uint32_t size,
8704                               uint64_t init_val)
8705 {
8706         struct regarray *r;
8707
8708         CHECK(p, EINVAL);
8709
8710         CHECK_NAME(name, EINVAL);
8711         CHECK(!regarray_find(p, name), EEXIST);
8712
8713         CHECK(size, EINVAL);
8714         size = rte_align32pow2(size);
8715
8716         /* Memory allocation. */
8717         r = calloc(1, sizeof(struct regarray));
8718         CHECK(r, ENOMEM);
8719
8720         /* Node initialization. */
8721         strcpy(r->name, name);
8722         r->init_val = init_val;
8723         r->size = size;
8724         r->id = p->n_regarrays;
8725
8726         /* Node add to tailq. */
8727         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8728         p->n_regarrays++;
8729
8730         return 0;
8731 }
8732
8733 static int
8734 regarray_build(struct rte_swx_pipeline *p)
8735 {
8736         struct regarray *regarray;
8737
8738         if (!p->n_regarrays)
8739                 return 0;
8740
8741         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8742         CHECK(p->regarray_runtime, ENOMEM);
8743
8744         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8745                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8746                 uint32_t i;
8747
8748                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8749                                          RTE_CACHE_LINE_SIZE,
8750                                          p->numa_node);
8751                 CHECK(r->regarray, ENOMEM);
8752
8753                 if (regarray->init_val)
8754                         for (i = 0; i < regarray->size; i++)
8755                                 r->regarray[i] = regarray->init_val;
8756
8757                 r->size_mask = regarray->size - 1;
8758         }
8759
8760         return 0;
8761 }
8762
8763 static void
8764 regarray_build_free(struct rte_swx_pipeline *p)
8765 {
8766         uint32_t i;
8767
8768         if (!p->regarray_runtime)
8769                 return;
8770
8771         for (i = 0; i < p->n_regarrays; i++) {
8772                 struct regarray *regarray = regarray_find_by_id(p, i);
8773                 struct regarray_runtime *r = &p->regarray_runtime[i];
8774
8775                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8776         }
8777
8778         free(p->regarray_runtime);
8779         p->regarray_runtime = NULL;
8780 }
8781
8782 static void
8783 regarray_free(struct rte_swx_pipeline *p)
8784 {
8785         regarray_build_free(p);
8786
8787         for ( ; ; ) {
8788                 struct regarray *elem;
8789
8790                 elem = TAILQ_FIRST(&p->regarrays);
8791                 if (!elem)
8792                         break;
8793
8794                 TAILQ_REMOVE(&p->regarrays, elem, node);
8795                 free(elem);
8796         }
8797 }
8798
8799 /*
8800  * Meter array.
8801  */
8802 static struct meter_profile *
8803 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8804 {
8805         struct meter_profile *elem;
8806
8807         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8808                 if (!strcmp(elem->name, name))
8809                         return elem;
8810
8811         return NULL;
8812 }
8813
8814 static struct metarray *
8815 metarray_find(struct rte_swx_pipeline *p, const char *name)
8816 {
8817         struct metarray *elem;
8818
8819         TAILQ_FOREACH(elem, &p->metarrays, node)
8820                 if (!strcmp(elem->name, name))
8821                         return elem;
8822
8823         return NULL;
8824 }
8825
8826 static struct metarray *
8827 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8828 {
8829         struct metarray *elem = NULL;
8830
8831         TAILQ_FOREACH(elem, &p->metarrays, node)
8832                 if (elem->id == id)
8833                         return elem;
8834
8835         return NULL;
8836 }
8837
8838 int
8839 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8840                                  const char *name,
8841                                  uint32_t size)
8842 {
8843         struct metarray *m;
8844
8845         CHECK(p, EINVAL);
8846
8847         CHECK_NAME(name, EINVAL);
8848         CHECK(!metarray_find(p, name), EEXIST);
8849
8850         CHECK(size, EINVAL);
8851         size = rte_align32pow2(size);
8852
8853         /* Memory allocation. */
8854         m = calloc(1, sizeof(struct metarray));
8855         CHECK(m, ENOMEM);
8856
8857         /* Node initialization. */
8858         strcpy(m->name, name);
8859         m->size = size;
8860         m->id = p->n_metarrays;
8861
8862         /* Node add to tailq. */
8863         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8864         p->n_metarrays++;
8865
8866         return 0;
8867 }
8868
8869 struct meter_profile meter_profile_default = {
8870         .node = {0},
8871         .name = "",
8872         .params = {0},
8873
8874         .profile = {
8875                 .cbs = 10000,
8876                 .pbs = 10000,
8877                 .cir_period = 1,
8878                 .cir_bytes_per_period = 1,
8879                 .pir_period = 1,
8880                 .pir_bytes_per_period = 1,
8881         },
8882
8883         .n_users = 0,
8884 };
8885
8886 static void
8887 meter_init(struct meter *m)
8888 {
8889         memset(m, 0, sizeof(struct meter));
8890         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8891         m->profile = &meter_profile_default;
8892         m->color_mask = RTE_COLOR_GREEN;
8893
8894         meter_profile_default.n_users++;
8895 }
8896
8897 static int
8898 metarray_build(struct rte_swx_pipeline *p)
8899 {
8900         struct metarray *m;
8901
8902         if (!p->n_metarrays)
8903                 return 0;
8904
8905         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8906         CHECK(p->metarray_runtime, ENOMEM);
8907
8908         TAILQ_FOREACH(m, &p->metarrays, node) {
8909                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8910                 uint32_t i;
8911
8912                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8913                                          RTE_CACHE_LINE_SIZE,
8914                                          p->numa_node);
8915                 CHECK(r->metarray, ENOMEM);
8916
8917                 for (i = 0; i < m->size; i++)
8918                         meter_init(&r->metarray[i]);
8919
8920                 r->size_mask = m->size - 1;
8921         }
8922
8923         return 0;
8924 }
8925
8926 static void
8927 metarray_build_free(struct rte_swx_pipeline *p)
8928 {
8929         uint32_t i;
8930
8931         if (!p->metarray_runtime)
8932                 return;
8933
8934         for (i = 0; i < p->n_metarrays; i++) {
8935                 struct metarray *m = metarray_find_by_id(p, i);
8936                 struct metarray_runtime *r = &p->metarray_runtime[i];
8937
8938                 env_free(r->metarray, m->size * sizeof(struct meter));
8939         }
8940
8941         free(p->metarray_runtime);
8942         p->metarray_runtime = NULL;
8943 }
8944
8945 static void
8946 metarray_free(struct rte_swx_pipeline *p)
8947 {
8948         metarray_build_free(p);
8949
8950         /* Meter arrays. */
8951         for ( ; ; ) {
8952                 struct metarray *elem;
8953
8954                 elem = TAILQ_FIRST(&p->metarrays);
8955                 if (!elem)
8956                         break;
8957
8958                 TAILQ_REMOVE(&p->metarrays, elem, node);
8959                 free(elem);
8960         }
8961
8962         /* Meter profiles. */
8963         for ( ; ; ) {
8964                 struct meter_profile *elem;
8965
8966                 elem = TAILQ_FIRST(&p->meter_profiles);
8967                 if (!elem)
8968                         break;
8969
8970                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8971                 free(elem);
8972         }
8973 }
8974
8975 /*
8976  * Pipeline.
8977  */
8978 int
8979 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8980 {
8981         struct rte_swx_pipeline *pipeline;
8982
8983         /* Check input parameters. */
8984         CHECK(p, EINVAL);
8985
8986         /* Memory allocation. */
8987         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8988         CHECK(pipeline, ENOMEM);
8989
8990         /* Initialization. */
8991         TAILQ_INIT(&pipeline->struct_types);
8992         TAILQ_INIT(&pipeline->port_in_types);
8993         TAILQ_INIT(&pipeline->ports_in);
8994         TAILQ_INIT(&pipeline->port_out_types);
8995         TAILQ_INIT(&pipeline->ports_out);
8996         TAILQ_INIT(&pipeline->extern_types);
8997         TAILQ_INIT(&pipeline->extern_objs);
8998         TAILQ_INIT(&pipeline->extern_funcs);
8999         TAILQ_INIT(&pipeline->headers);
9000         TAILQ_INIT(&pipeline->actions);
9001         TAILQ_INIT(&pipeline->table_types);
9002         TAILQ_INIT(&pipeline->tables);
9003         TAILQ_INIT(&pipeline->selectors);
9004         TAILQ_INIT(&pipeline->learners);
9005         TAILQ_INIT(&pipeline->regarrays);
9006         TAILQ_INIT(&pipeline->meter_profiles);
9007         TAILQ_INIT(&pipeline->metarrays);
9008
9009         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9010         pipeline->numa_node = numa_node;
9011
9012         *p = pipeline;
9013         return 0;
9014 }
9015
9016 void
9017 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9018 {
9019         void *lib;
9020
9021         if (!p)
9022                 return;
9023
9024         lib = p->lib;
9025
9026         free(p->instruction_data);
9027         free(p->instructions);
9028
9029         metarray_free(p);
9030         regarray_free(p);
9031         table_state_free(p);
9032         learner_free(p);
9033         selector_free(p);
9034         table_free(p);
9035         action_free(p);
9036         instruction_table_free(p);
9037         metadata_free(p);
9038         header_free(p);
9039         extern_func_free(p);
9040         extern_obj_free(p);
9041         port_out_free(p);
9042         port_in_free(p);
9043         struct_free(p);
9044
9045         free(p);
9046
9047         if (lib)
9048                 dlclose(lib);
9049 }
9050
9051 int
9052 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9053                                      const char **instructions,
9054                                      uint32_t n_instructions)
9055 {
9056         int err;
9057         uint32_t i;
9058
9059         err = instruction_config(p, NULL, instructions, n_instructions);
9060         if (err)
9061                 return err;
9062
9063         /* Thread instruction pointer reset. */
9064         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9065                 struct thread *t = &p->threads[i];
9066
9067                 thread_ip_reset(p, t);
9068         }
9069
9070         return 0;
9071 }
9072
9073 static int
9074 pipeline_compile(struct rte_swx_pipeline *p);
9075
9076 int
9077 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9078 {
9079         int status;
9080
9081         CHECK(p, EINVAL);
9082         CHECK(p->build_done == 0, EEXIST);
9083
9084         status = port_in_build(p);
9085         if (status)
9086                 goto error;
9087
9088         status = port_out_build(p);
9089         if (status)
9090                 goto error;
9091
9092         status = struct_build(p);
9093         if (status)
9094                 goto error;
9095
9096         status = extern_obj_build(p);
9097         if (status)
9098                 goto error;
9099
9100         status = extern_func_build(p);
9101         if (status)
9102                 goto error;
9103
9104         status = header_build(p);
9105         if (status)
9106                 goto error;
9107
9108         status = metadata_build(p);
9109         if (status)
9110                 goto error;
9111
9112         status = instruction_table_build(p);
9113         if (status)
9114                 goto error;
9115
9116         status = action_build(p);
9117         if (status)
9118                 goto error;
9119
9120         status = table_build(p);
9121         if (status)
9122                 goto error;
9123
9124         status = selector_build(p);
9125         if (status)
9126                 goto error;
9127
9128         status = learner_build(p);
9129         if (status)
9130                 goto error;
9131
9132         status = table_state_build(p);
9133         if (status)
9134                 goto error;
9135
9136         status = regarray_build(p);
9137         if (status)
9138                 goto error;
9139
9140         status = metarray_build(p);
9141         if (status)
9142                 goto error;
9143
9144         p->build_done = 1;
9145
9146         pipeline_compile(p);
9147
9148         return 0;
9149
9150 error:
9151         metarray_build_free(p);
9152         regarray_build_free(p);
9153         table_state_build_free(p);
9154         learner_build_free(p);
9155         selector_build_free(p);
9156         table_build_free(p);
9157         action_build_free(p);
9158         instruction_table_build_free(p);
9159         metadata_build_free(p);
9160         header_build_free(p);
9161         extern_func_build_free(p);
9162         extern_obj_build_free(p);
9163         port_out_build_free(p);
9164         port_in_build_free(p);
9165         struct_build_free(p);
9166
9167         return status;
9168 }
9169
9170 void
9171 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9172 {
9173         uint32_t i;
9174
9175         for (i = 0; i < n_instructions; i++)
9176                 instr_exec(p);
9177 }
9178
9179 void
9180 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9181 {
9182         uint32_t i;
9183
9184         for (i = 0; i < p->n_ports_out; i++) {
9185                 struct port_out_runtime *port = &p->out[i];
9186
9187                 if (port->flush)
9188                         port->flush(port->obj);
9189         }
9190 }
9191
9192 /*
9193  * Control.
9194  */
9195 int
9196 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9197                               struct rte_swx_ctl_pipeline_info *pipeline)
9198 {
9199         struct action *action;
9200         struct table *table;
9201         uint32_t n_actions = 0, n_tables = 0;
9202
9203         if (!p || !pipeline)
9204                 return -EINVAL;
9205
9206         TAILQ_FOREACH(action, &p->actions, node)
9207                 n_actions++;
9208
9209         TAILQ_FOREACH(table, &p->tables, node)
9210                 n_tables++;
9211
9212         pipeline->n_ports_in = p->n_ports_in;
9213         pipeline->n_ports_out = p->n_ports_out;
9214         pipeline->n_actions = n_actions;
9215         pipeline->n_tables = n_tables;
9216         pipeline->n_selectors = p->n_selectors;
9217         pipeline->n_learners = p->n_learners;
9218         pipeline->n_regarrays = p->n_regarrays;
9219         pipeline->n_metarrays = p->n_metarrays;
9220
9221         return 0;
9222 }
9223
9224 int
9225 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9226 {
9227         if (!p || !numa_node)
9228                 return -EINVAL;
9229
9230         *numa_node = p->numa_node;
9231         return 0;
9232 }
9233
9234 int
9235 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9236                             uint32_t action_id,
9237                             struct rte_swx_ctl_action_info *action)
9238 {
9239         struct action *a = NULL;
9240
9241         if (!p || (action_id >= p->n_actions) || !action)
9242                 return -EINVAL;
9243
9244         a = action_find_by_id(p, action_id);
9245         if (!a)
9246                 return -EINVAL;
9247
9248         strcpy(action->name, a->name);
9249         action->n_args = a->st ? a->st->n_fields : 0;
9250         return 0;
9251 }
9252
9253 int
9254 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9255                                 uint32_t action_id,
9256                                 uint32_t action_arg_id,
9257                                 struct rte_swx_ctl_action_arg_info *action_arg)
9258 {
9259         struct action *a = NULL;
9260         struct field *arg = NULL;
9261
9262         if (!p || (action_id >= p->n_actions) || !action_arg)
9263                 return -EINVAL;
9264
9265         a = action_find_by_id(p, action_id);
9266         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9267                 return -EINVAL;
9268
9269         arg = &a->st->fields[action_arg_id];
9270         strcpy(action_arg->name, arg->name);
9271         action_arg->n_bits = arg->n_bits;
9272         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9273
9274         return 0;
9275 }
9276
9277 int
9278 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9279                            uint32_t table_id,
9280                            struct rte_swx_ctl_table_info *table)
9281 {
9282         struct table *t = NULL;
9283
9284         if (!p || !table)
9285                 return -EINVAL;
9286
9287         t = table_find_by_id(p, table_id);
9288         if (!t)
9289                 return -EINVAL;
9290
9291         strcpy(table->name, t->name);
9292         strcpy(table->args, t->args);
9293         table->n_match_fields = t->n_fields;
9294         table->n_actions = t->n_actions;
9295         table->default_action_is_const = t->default_action_is_const;
9296         table->size = t->size;
9297         return 0;
9298 }
9299
9300 int
9301 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9302         uint32_t table_id,
9303         uint32_t match_field_id,
9304         struct rte_swx_ctl_table_match_field_info *match_field)
9305 {
9306         struct table *t;
9307         struct match_field *f;
9308
9309         if (!p || (table_id >= p->n_tables) || !match_field)
9310                 return -EINVAL;
9311
9312         t = table_find_by_id(p, table_id);
9313         if (!t || (match_field_id >= t->n_fields))
9314                 return -EINVAL;
9315
9316         f = &t->fields[match_field_id];
9317         match_field->match_type = f->match_type;
9318         match_field->is_header = t->header ? 1 : 0;
9319         match_field->n_bits = f->field->n_bits;
9320         match_field->offset = f->field->offset;
9321
9322         return 0;
9323 }
9324
9325 int
9326 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9327         uint32_t table_id,
9328         uint32_t table_action_id,
9329         struct rte_swx_ctl_table_action_info *table_action)
9330 {
9331         struct table *t;
9332
9333         if (!p || (table_id >= p->n_tables) || !table_action)
9334                 return -EINVAL;
9335
9336         t = table_find_by_id(p, table_id);
9337         if (!t || (table_action_id >= t->n_actions))
9338                 return -EINVAL;
9339
9340         table_action->action_id = t->actions[table_action_id]->id;
9341
9342         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9343         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9344
9345         return 0;
9346 }
9347
9348 int
9349 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9350                           uint32_t table_id,
9351                           struct rte_swx_table_ops *table_ops,
9352                           int *is_stub)
9353 {
9354         struct table *t;
9355
9356         if (!p || (table_id >= p->n_tables))
9357                 return -EINVAL;
9358
9359         t = table_find_by_id(p, table_id);
9360         if (!t)
9361                 return -EINVAL;
9362
9363         if (t->type) {
9364                 if (table_ops)
9365                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9366                 *is_stub = 0;
9367         } else {
9368                 *is_stub = 1;
9369         }
9370
9371         return 0;
9372 }
9373
9374 int
9375 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9376                               uint32_t selector_id,
9377                               struct rte_swx_ctl_selector_info *selector)
9378 {
9379         struct selector *s = NULL;
9380
9381         if (!p || !selector)
9382                 return -EINVAL;
9383
9384         s = selector_find_by_id(p, selector_id);
9385         if (!s)
9386                 return -EINVAL;
9387
9388         strcpy(selector->name, s->name);
9389
9390         selector->n_selector_fields = s->n_selector_fields;
9391         selector->n_groups_max = s->n_groups_max;
9392         selector->n_members_per_group_max = s->n_members_per_group_max;
9393
9394         return 0;
9395 }
9396
9397 int
9398 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9399          uint32_t selector_id,
9400          struct rte_swx_ctl_table_match_field_info *field)
9401 {
9402         struct selector *s;
9403
9404         if (!p || (selector_id >= p->n_selectors) || !field)
9405                 return -EINVAL;
9406
9407         s = selector_find_by_id(p, selector_id);
9408         if (!s)
9409                 return -EINVAL;
9410
9411         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9412         field->is_header = 0;
9413         field->n_bits = s->group_id_field->n_bits;
9414         field->offset = s->group_id_field->offset;
9415
9416         return 0;
9417 }
9418
9419 int
9420 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9421          uint32_t selector_id,
9422          uint32_t selector_field_id,
9423          struct rte_swx_ctl_table_match_field_info *field)
9424 {
9425         struct selector *s;
9426         struct field *f;
9427
9428         if (!p || (selector_id >= p->n_selectors) || !field)
9429                 return -EINVAL;
9430
9431         s = selector_find_by_id(p, selector_id);
9432         if (!s || (selector_field_id >= s->n_selector_fields))
9433                 return -EINVAL;
9434
9435         f = s->selector_fields[selector_field_id];
9436         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9437         field->is_header = s->selector_header ? 1 : 0;
9438         field->n_bits = f->n_bits;
9439         field->offset = f->offset;
9440
9441         return 0;
9442 }
9443
9444 int
9445 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9446          uint32_t selector_id,
9447          struct rte_swx_ctl_table_match_field_info *field)
9448 {
9449         struct selector *s;
9450
9451         if (!p || (selector_id >= p->n_selectors) || !field)
9452                 return -EINVAL;
9453
9454         s = selector_find_by_id(p, selector_id);
9455         if (!s)
9456                 return -EINVAL;
9457
9458         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9459         field->is_header = 0;
9460         field->n_bits = s->member_id_field->n_bits;
9461         field->offset = s->member_id_field->offset;
9462
9463         return 0;
9464 }
9465
9466 int
9467 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9468                              uint32_t learner_id,
9469                              struct rte_swx_ctl_learner_info *learner)
9470 {
9471         struct learner *l = NULL;
9472
9473         if (!p || !learner)
9474                 return -EINVAL;
9475
9476         l = learner_find_by_id(p, learner_id);
9477         if (!l)
9478                 return -EINVAL;
9479
9480         strcpy(learner->name, l->name);
9481
9482         learner->n_match_fields = l->n_fields;
9483         learner->n_actions = l->n_actions;
9484         learner->default_action_is_const = l->default_action_is_const;
9485         learner->size = l->size;
9486
9487         return 0;
9488 }
9489
9490 int
9491 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9492                                          uint32_t learner_id,
9493                                          uint32_t match_field_id,
9494                                          struct rte_swx_ctl_table_match_field_info *match_field)
9495 {
9496         struct learner *l;
9497         struct field *f;
9498
9499         if (!p || (learner_id >= p->n_learners) || !match_field)
9500                 return -EINVAL;
9501
9502         l = learner_find_by_id(p, learner_id);
9503         if (!l || (match_field_id >= l->n_fields))
9504                 return -EINVAL;
9505
9506         f = l->fields[match_field_id];
9507         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9508         match_field->is_header = l->header ? 1 : 0;
9509         match_field->n_bits = f->n_bits;
9510         match_field->offset = f->offset;
9511
9512         return 0;
9513 }
9514
9515 int
9516 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9517                                     uint32_t learner_id,
9518                                     uint32_t learner_action_id,
9519                                     struct rte_swx_ctl_table_action_info *learner_action)
9520 {
9521         struct learner *l;
9522
9523         if (!p || (learner_id >= p->n_learners) || !learner_action)
9524                 return -EINVAL;
9525
9526         l = learner_find_by_id(p, learner_id);
9527         if (!l || (learner_action_id >= l->n_actions))
9528                 return -EINVAL;
9529
9530         learner_action->action_id = l->actions[learner_action_id]->id;
9531
9532         learner_action->action_is_for_table_entries =
9533                 l->action_is_for_table_entries[learner_action_id];
9534
9535         learner_action->action_is_for_default_entry =
9536                 l->action_is_for_default_entry[learner_action_id];
9537
9538         return 0;
9539 }
9540
9541 int
9542 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9543                                  struct rte_swx_table_state **table_state)
9544 {
9545         if (!p || !table_state || !p->build_done)
9546                 return -EINVAL;
9547
9548         *table_state = p->table_state;
9549         return 0;
9550 }
9551
9552 int
9553 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9554                                  struct rte_swx_table_state *table_state)
9555 {
9556         if (!p || !table_state || !p->build_done)
9557                 return -EINVAL;
9558
9559         p->table_state = table_state;
9560         return 0;
9561 }
9562
9563 int
9564 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9565                                         uint32_t port_id,
9566                                         struct rte_swx_port_in_stats *stats)
9567 {
9568         struct port_in *port;
9569
9570         if (!p || !stats)
9571                 return -EINVAL;
9572
9573         port = port_in_find(p, port_id);
9574         if (!port)
9575                 return -EINVAL;
9576
9577         port->type->ops.stats_read(port->obj, stats);
9578         return 0;
9579 }
9580
9581 int
9582 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9583                                          uint32_t port_id,
9584                                          struct rte_swx_port_out_stats *stats)
9585 {
9586         struct port_out *port;
9587
9588         if (!p || !stats)
9589                 return -EINVAL;
9590
9591         port = port_out_find(p, port_id);
9592         if (!port)
9593                 return -EINVAL;
9594
9595         port->type->ops.stats_read(port->obj, stats);
9596         return 0;
9597 }
9598
9599 int
9600 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9601                                       const char *table_name,
9602                                       struct rte_swx_table_stats *stats)
9603 {
9604         struct table *table;
9605         struct table_statistics *table_stats;
9606
9607         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9608                 return -EINVAL;
9609
9610         table = table_find(p, table_name);
9611         if (!table)
9612                 return -EINVAL;
9613
9614         table_stats = &p->table_stats[table->id];
9615
9616         memcpy(stats->n_pkts_action,
9617                table_stats->n_pkts_action,
9618                p->n_actions * sizeof(uint64_t));
9619
9620         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9621         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9622
9623         return 0;
9624 }
9625
9626 int
9627 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9628         const char *selector_name,
9629         struct rte_swx_pipeline_selector_stats *stats)
9630 {
9631         struct selector *s;
9632
9633         if (!p || !selector_name || !selector_name[0] || !stats)
9634                 return -EINVAL;
9635
9636         s = selector_find(p, selector_name);
9637         if (!s)
9638                 return -EINVAL;
9639
9640         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9641
9642         return 0;
9643 }
9644
9645 int
9646 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9647                                         const char *learner_name,
9648                                         struct rte_swx_learner_stats *stats)
9649 {
9650         struct learner *l;
9651         struct learner_statistics *learner_stats;
9652
9653         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9654                 return -EINVAL;
9655
9656         l = learner_find(p, learner_name);
9657         if (!l)
9658                 return -EINVAL;
9659
9660         learner_stats = &p->learner_stats[l->id];
9661
9662         memcpy(stats->n_pkts_action,
9663                learner_stats->n_pkts_action,
9664                p->n_actions * sizeof(uint64_t));
9665
9666         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9667         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9668
9669         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9670         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9671
9672         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9673
9674         return 0;
9675 }
9676
9677 int
9678 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9679                               uint32_t regarray_id,
9680                               struct rte_swx_ctl_regarray_info *regarray)
9681 {
9682         struct regarray *r;
9683
9684         if (!p || !regarray)
9685                 return -EINVAL;
9686
9687         r = regarray_find_by_id(p, regarray_id);
9688         if (!r)
9689                 return -EINVAL;
9690
9691         strcpy(regarray->name, r->name);
9692         regarray->size = r->size;
9693         return 0;
9694 }
9695
9696 int
9697 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9698                                    const char *regarray_name,
9699                                    uint32_t regarray_index,
9700                                    uint64_t *value)
9701 {
9702         struct regarray *regarray;
9703         struct regarray_runtime *r;
9704
9705         if (!p || !regarray_name || !value)
9706                 return -EINVAL;
9707
9708         regarray = regarray_find(p, regarray_name);
9709         if (!regarray || (regarray_index >= regarray->size))
9710                 return -EINVAL;
9711
9712         r = &p->regarray_runtime[regarray->id];
9713         *value = r->regarray[regarray_index];
9714         return 0;
9715 }
9716
9717 int
9718 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9719                                    const char *regarray_name,
9720                                    uint32_t regarray_index,
9721                                    uint64_t value)
9722 {
9723         struct regarray *regarray;
9724         struct regarray_runtime *r;
9725
9726         if (!p || !regarray_name)
9727                 return -EINVAL;
9728
9729         regarray = regarray_find(p, regarray_name);
9730         if (!regarray || (regarray_index >= regarray->size))
9731                 return -EINVAL;
9732
9733         r = &p->regarray_runtime[regarray->id];
9734         r->regarray[regarray_index] = value;
9735         return 0;
9736 }
9737
9738 int
9739 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9740                               uint32_t metarray_id,
9741                               struct rte_swx_ctl_metarray_info *metarray)
9742 {
9743         struct metarray *m;
9744
9745         if (!p || !metarray)
9746                 return -EINVAL;
9747
9748         m = metarray_find_by_id(p, metarray_id);
9749         if (!m)
9750                 return -EINVAL;
9751
9752         strcpy(metarray->name, m->name);
9753         metarray->size = m->size;
9754         return 0;
9755 }
9756
9757 int
9758 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9759                               const char *name,
9760                               struct rte_meter_trtcm_params *params)
9761 {
9762         struct meter_profile *mp;
9763         int status;
9764
9765         CHECK(p, EINVAL);
9766         CHECK_NAME(name, EINVAL);
9767         CHECK(params, EINVAL);
9768         CHECK(!meter_profile_find(p, name), EEXIST);
9769
9770         /* Node allocation. */
9771         mp = calloc(1, sizeof(struct meter_profile));
9772         CHECK(mp, ENOMEM);
9773
9774         /* Node initialization. */
9775         strcpy(mp->name, name);
9776         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9777         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9778         if (status) {
9779                 free(mp);
9780                 CHECK(0, EINVAL);
9781         }
9782
9783         /* Node add to tailq. */
9784         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9785
9786         return 0;
9787 }
9788
9789 int
9790 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9791                                  const char *name)
9792 {
9793         struct meter_profile *mp;
9794
9795         CHECK(p, EINVAL);
9796         CHECK_NAME(name, EINVAL);
9797
9798         mp = meter_profile_find(p, name);
9799         CHECK(mp, EINVAL);
9800         CHECK(!mp->n_users, EBUSY);
9801
9802         /* Remove node from tailq. */
9803         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9804         free(mp);
9805
9806         return 0;
9807 }
9808
9809 int
9810 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9811                         const char *metarray_name,
9812                         uint32_t metarray_index)
9813 {
9814         struct meter_profile *mp_old;
9815         struct metarray *metarray;
9816         struct metarray_runtime *metarray_runtime;
9817         struct meter *m;
9818
9819         CHECK(p, EINVAL);
9820         CHECK_NAME(metarray_name, EINVAL);
9821
9822         metarray = metarray_find(p, metarray_name);
9823         CHECK(metarray, EINVAL);
9824         CHECK(metarray_index < metarray->size, EINVAL);
9825
9826         metarray_runtime = &p->metarray_runtime[metarray->id];
9827         m = &metarray_runtime->metarray[metarray_index];
9828         mp_old = m->profile;
9829
9830         meter_init(m);
9831
9832         mp_old->n_users--;
9833
9834         return 0;
9835 }
9836
9837 int
9838 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9839                       const char *metarray_name,
9840                       uint32_t metarray_index,
9841                       const char *profile_name)
9842 {
9843         struct meter_profile *mp, *mp_old;
9844         struct metarray *metarray;
9845         struct metarray_runtime *metarray_runtime;
9846         struct meter *m;
9847
9848         CHECK(p, EINVAL);
9849         CHECK_NAME(metarray_name, EINVAL);
9850
9851         metarray = metarray_find(p, metarray_name);
9852         CHECK(metarray, EINVAL);
9853         CHECK(metarray_index < metarray->size, EINVAL);
9854
9855         mp = meter_profile_find(p, profile_name);
9856         CHECK(mp, EINVAL);
9857
9858         metarray_runtime = &p->metarray_runtime[metarray->id];
9859         m = &metarray_runtime->metarray[metarray_index];
9860         mp_old = m->profile;
9861
9862         memset(m, 0, sizeof(struct meter));
9863         rte_meter_trtcm_config(&m->m, &mp->profile);
9864         m->profile = mp;
9865         m->color_mask = RTE_COLORS;
9866
9867         mp->n_users++;
9868         mp_old->n_users--;
9869
9870         return 0;
9871 }
9872
9873 int
9874 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9875                              const char *metarray_name,
9876                              uint32_t metarray_index,
9877                              struct rte_swx_ctl_meter_stats *stats)
9878 {
9879         struct metarray *metarray;
9880         struct metarray_runtime *metarray_runtime;
9881         struct meter *m;
9882
9883         CHECK(p, EINVAL);
9884         CHECK_NAME(metarray_name, EINVAL);
9885
9886         metarray = metarray_find(p, metarray_name);
9887         CHECK(metarray, EINVAL);
9888         CHECK(metarray_index < metarray->size, EINVAL);
9889
9890         CHECK(stats, EINVAL);
9891
9892         metarray_runtime = &p->metarray_runtime[metarray->id];
9893         m = &metarray_runtime->metarray[metarray_index];
9894
9895         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9896         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9897
9898         return 0;
9899 }
9900
9901 /*
9902  * Pipeline compilation.
9903  */
9904 static const char *
9905 instr_type_to_name(struct instruction *instr)
9906 {
9907         switch (instr->type) {
9908         case INSTR_RX: return "INSTR_RX";
9909
9910         case INSTR_TX: return "INSTR_TX";
9911         case INSTR_TX_I: return "INSTR_TX_I";
9912
9913         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
9914         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
9915         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
9916         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
9917         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
9918         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
9919         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
9920         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
9921
9922         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
9923
9924         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
9925
9926         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
9927         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
9928         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
9929         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
9930         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
9931         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
9932         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
9933         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
9934         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
9935
9936         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
9937         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
9938
9939         case INSTR_MOV: return "INSTR_MOV";
9940         case INSTR_MOV_MH: return "INSTR_MOV_MH";
9941         case INSTR_MOV_HM: return "INSTR_MOV_HM";
9942         case INSTR_MOV_HH: return "INSTR_MOV_HH";
9943         case INSTR_MOV_I: return "INSTR_MOV_I";
9944
9945         case INSTR_DMA_HT: return "INSTR_DMA_HT";
9946         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
9947         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
9948         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
9949         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
9950         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
9951         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
9952         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
9953
9954         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
9955         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
9956         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
9957         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
9958         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
9959         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
9960
9961         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
9962         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
9963         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
9964         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
9965         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
9966         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
9967
9968         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
9969         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
9970         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
9971         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
9972
9973         case INSTR_ALU_AND: return "INSTR_ALU_AND";
9974         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
9975         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
9976         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
9977         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
9978
9979         case INSTR_ALU_OR: return "INSTR_ALU_OR";
9980         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
9981         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
9982         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
9983         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
9984
9985         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
9986         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
9987         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
9988         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
9989         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
9990
9991         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
9992         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
9993         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
9994         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
9995         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
9996         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
9997
9998         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
9999         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10000         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10001         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10002         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10003         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10004
10005         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10006         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10007         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10008
10009         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10010         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10011         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10012         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10013         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10014         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10015
10016         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10017         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10018         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10019         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10020         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10021         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10022         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10023         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10024         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10025
10026         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10027         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10028         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10029         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10030         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10031         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10032         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10033         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10034         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10035
10036         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10037         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10038         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10039
10040         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10041         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10042         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10043         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10044         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10045         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10046         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10047         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10048         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10049         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10050         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10051         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10052
10053         case INSTR_TABLE: return "INSTR_TABLE";
10054         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10055         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10056         case INSTR_LEARNER: return "INSTR_LEARNER";
10057         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10058
10059         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10060         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10061
10062         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10063         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10064
10065         case INSTR_JMP: return "INSTR_JMP";
10066         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10067         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10068         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10069         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10070         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10071         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10072         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10073         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10074         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10075         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10076         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10077         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10078         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10079         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10080         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10081         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10082         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10083         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10084         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10085         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10086         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10087         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10088         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10089         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10090         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10091         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10092         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10093         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10094
10095         case INSTR_RETURN: return "INSTR_RETURN";
10096
10097         default: return "INSTR_UNKNOWN";
10098         }
10099 }
10100
10101 typedef void
10102 (*instruction_export_t)(struct instruction *, FILE *);
10103
10104 static void
10105 instr_io_export(struct instruction *instr, FILE *f)
10106 {
10107         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10108
10109         /* n_io, n_io_imm, n_hdrs. */
10110         if (instr->type == INSTR_RX ||
10111             instr->type == INSTR_TX ||
10112             instr->type == INSTR_HDR_EXTRACT_M ||
10113             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10114                 n_io = 1;
10115
10116         if (instr->type == INSTR_TX_I)
10117                 n_io_imm = 1;
10118
10119         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10120                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10121
10122         if (instr->type == INSTR_HDR_EXTRACT_M ||
10123             instr->type == INSTR_HDR_LOOKAHEAD ||
10124             instr->type == INSTR_HDR_EMIT)
10125                 n_hdrs = 1;
10126
10127         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10128                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10129
10130         /* instr. */
10131         fprintf(f,
10132                 "\t{\n"
10133                 "\t\t.type = %s,\n",
10134                 instr_type_to_name(instr));
10135
10136         /* instr.io. */
10137         fprintf(f,
10138                 "\t\t.io = {\n");
10139
10140         /* instr.io.io. */
10141         if (n_io)
10142                 fprintf(f,
10143                         "\t\t\t.io = {\n"
10144                         "\t\t\t\t.offset = %u,\n"
10145                         "\t\t\t\t.n_bits = %u,\n"
10146                         "\t\t\t},\n",
10147                         instr->io.io.offset,
10148                         instr->io.io.n_bits);
10149
10150         if (n_io_imm)
10151                 fprintf(f,
10152                         "\t\t\t.io = {\n"
10153                         "\t\t\t\t.val = %u,\n"
10154                         "\t\t\t},\n",
10155                         instr->io.io.val);
10156
10157         /* instr.io.hdr. */
10158         if (n_hdrs) {
10159                 fprintf(f,
10160                         "\t\t.hdr = {\n");
10161
10162                 /* instr.io.hdr.header_id. */
10163                 fprintf(f,
10164                         "\t\t\t.header_id = {");
10165
10166                 for (i = 0; i < n_hdrs; i++)
10167                         fprintf(f,
10168                                 "%u, ",
10169                                 instr->io.hdr.header_id[i]);
10170
10171                 fprintf(f,
10172                         "},\n");
10173
10174                 /* instr.io.hdr.struct_id. */
10175                 fprintf(f,
10176                         "\t\t\t.struct_id = {");
10177
10178                 for (i = 0; i < n_hdrs; i++)
10179                         fprintf(f,
10180                                 "%u, ",
10181                                 instr->io.hdr.struct_id[i]);
10182
10183                 fprintf(f,
10184                         "},\n");
10185
10186                 /* instr.io.hdr.n_bytes. */
10187                 fprintf(f,
10188                         "\t\t\t.n_bytes = {");
10189
10190                 for (i = 0; i < n_hdrs; i++)
10191                         fprintf(f,
10192                                 "%u, ",
10193                                 instr->io.hdr.n_bytes[i]);
10194
10195                 fprintf(f,
10196                         "},\n");
10197
10198                 /* instr.io.hdr - closing curly brace. */
10199                 fprintf(f,
10200                         "\t\t\t}\n,");
10201         }
10202
10203         /* instr.io - closing curly brace. */
10204         fprintf(f,
10205                 "\t\t},\n");
10206
10207         /* instr - closing curly brace. */
10208         fprintf(f,
10209                 "\t},\n");
10210 }
10211
10212 static void
10213 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10214 {
10215         fprintf(f,
10216                 "\t{\n"
10217                 "\t\t.type = %s,\n"
10218                 "\t\t.valid = {\n"
10219                 "\t\t\t.header_id = %u,\n"
10220                 "\t\t},\n"
10221                 "\t},\n",
10222                 instr_type_to_name(instr),
10223                 instr->valid.header_id);
10224 }
10225
10226 static void
10227 instr_mov_export(struct instruction *instr, FILE *f)
10228 {
10229         if (instr->type != INSTR_MOV_I)
10230                 fprintf(f,
10231                         "\t{\n"
10232                         "\t\t.type = %s,\n"
10233                         "\t\t.mov = {\n"
10234                         "\t\t\t.dst = {\n"
10235                         "\t\t\t\t.struct_id = %u,\n"
10236                         "\t\t\t\t.n_bits = %u,\n"
10237                         "\t\t\t\t.offset = %u,\n"
10238                         "\t\t\t},\n"
10239                         "\t\t\t.src = {\n"
10240                         "\t\t\t\t.struct_id = %u,\n"
10241                         "\t\t\t\t.n_bits = %u,\n"
10242                         "\t\t\t\t.offset = %u,\n"
10243                         "\t\t\t},\n"
10244                         "\t\t},\n"
10245                         "\t},\n",
10246                         instr_type_to_name(instr),
10247                         instr->mov.dst.struct_id,
10248                         instr->mov.dst.n_bits,
10249                         instr->mov.dst.offset,
10250                         instr->mov.src.struct_id,
10251                         instr->mov.src.n_bits,
10252                         instr->mov.src.offset);
10253         else
10254                 fprintf(f,
10255                         "\t{\n"
10256                         "\t\t.type = %s,\n"
10257                         "\t\t.mov = {\n"
10258                         "\t\t\t.dst = {\n"
10259                         "\t\t\t\t.struct_id = %u,\n"
10260                         "\t\t\t\t.n_bits = %u,\n"
10261                         "\t\t\t\t.offset = %u,\n"
10262                         "\t\t\t}\n,"
10263                         "\t\t\t.src_val = %" PRIu64 ",\n"
10264                         "\t\t},\n"
10265                         "\t},\n",
10266                         instr_type_to_name(instr),
10267                         instr->mov.dst.struct_id,
10268                         instr->mov.dst.n_bits,
10269                         instr->mov.dst.offset,
10270                         instr->mov.src_val);
10271 }
10272
10273 static void
10274 instr_dma_ht_export(struct instruction *instr, FILE *f)
10275 {
10276         uint32_t n_dma = 0, i;
10277
10278         /* n_dma. */
10279         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10280
10281         /* instr. */
10282         fprintf(f,
10283                 "\t{\n"
10284                 "\t\t.type = %s,\n",
10285                 instr_type_to_name(instr));
10286
10287         /* instr.dma. */
10288         fprintf(f,
10289                 "\t\t.dma = {\n");
10290
10291         /* instr.dma.dst. */
10292         fprintf(f,
10293                 "\t\t\t.dst = {\n");
10294
10295         /* instr.dma.dst.header_id. */
10296         fprintf(f,
10297                 "\t\t\t\t.header_id = {");
10298
10299         for (i = 0; i < n_dma; i++)
10300                 fprintf(f,
10301                         "%u, ",
10302                         instr->dma.dst.header_id[i]);
10303
10304         fprintf(f,
10305                 "},\n");
10306
10307         /* instr.dma.dst.struct_id. */
10308         fprintf(f,
10309                 "\t\t\t\t.struct_id = {");
10310
10311         for (i = 0; i < n_dma; i++)
10312                 fprintf(f,
10313                         "%u, ",
10314                         instr->dma.dst.struct_id[i]);
10315
10316         fprintf(f,
10317                 "},\n");
10318
10319         /* instr.dma.dst - closing curly brace. */
10320         fprintf(f,
10321                 "\t\t\t},\n");
10322
10323         /* instr.dma.src. */
10324         fprintf(f,
10325                 "\t\t\t.src = {\n");
10326
10327         /* instr.dma.src.offset. */
10328         fprintf(f,
10329                 "\t\t\t\t.offset = {");
10330
10331         for (i = 0; i < n_dma; i++)
10332                 fprintf(f,
10333                         "%u, ",
10334                         instr->dma.src.offset[i]);
10335
10336         fprintf(f,
10337                 "},\n");
10338
10339         /* instr.dma.src - closing curly brace. */
10340         fprintf(f,
10341                 "\t\t\t},\n");
10342
10343         /* instr.dma.n_bytes. */
10344         fprintf(f,
10345                 "\t\t\t.n_bytes = {");
10346
10347         for (i = 0; i < n_dma; i++)
10348                 fprintf(f,
10349                         "%u, ",
10350                         instr->dma.n_bytes[i]);
10351
10352         fprintf(f,
10353                 "},\n");
10354
10355         /* instr.dma - closing curly brace. */
10356         fprintf(f,
10357                 "\t\t},\n");
10358
10359         /* instr - closing curly brace. */
10360         fprintf(f,
10361                 "\t},\n");
10362 }
10363
10364 static void
10365 instr_alu_export(struct instruction *instr, FILE *f)
10366 {
10367         int imm = 0;
10368
10369         if (instr->type == INSTR_ALU_ADD_MI ||
10370             instr->type == INSTR_ALU_ADD_HI ||
10371             instr->type == INSTR_ALU_SUB_MI ||
10372             instr->type == INSTR_ALU_SUB_HI ||
10373             instr->type == INSTR_ALU_SHL_MI ||
10374             instr->type == INSTR_ALU_SHL_HI ||
10375             instr->type == INSTR_ALU_SHR_MI ||
10376             instr->type == INSTR_ALU_SHR_HI ||
10377             instr->type == INSTR_ALU_AND_I ||
10378             instr->type == INSTR_ALU_OR_I ||
10379             instr->type == INSTR_ALU_XOR_I)
10380                 imm = 1;
10381
10382         if (!imm)
10383                 fprintf(f,
10384                         "\t{\n"
10385                         "\t\t.type = %s,\n"
10386                         "\t\t.alu = {\n"
10387                         "\t\t\t.dst = {\n"
10388                         "\t\t\t\t.struct_id = %u,\n"
10389                         "\t\t\t\t.n_bits = %u,\n"
10390                         "\t\t\t\t.offset = %u,\n"
10391                         "\t\t\t},\n"
10392                         "\t\t\t.src = {\n"
10393                         "\t\t\t\t.struct_id = %u,\n"
10394                         "\t\t\t\t.n_bits = %u,\n"
10395                         "\t\t\t\t.offset = %u,\n"
10396                         "\t\t\t},\n"
10397                         "\t\t},\n"
10398                         "\t},\n",
10399                         instr_type_to_name(instr),
10400                         instr->alu.dst.struct_id,
10401                         instr->alu.dst.n_bits,
10402                         instr->alu.dst.offset,
10403                         instr->alu.src.struct_id,
10404                         instr->alu.src.n_bits,
10405                         instr->alu.src.offset);
10406         else
10407                 fprintf(f,
10408                         "\t{\n"
10409                         "\t\t.type = %s,\n"
10410                         "\t\t.alu = {\n"
10411                         "\t\t\t.dst = {\n"
10412                         "\t\t\t\t.struct_id = %u,\n"
10413                         "\t\t\t\t.n_bits = %u,\n"
10414                         "\t\t\t\t.offset = %u,\n"
10415                         "\t\t\t}\n,"
10416                         "\t\t\t.src_val = %" PRIu64 ",\n"
10417                         "\t\t},\n"
10418                         "\t},\n",
10419                         instr_type_to_name(instr),
10420                         instr->alu.dst.struct_id,
10421                         instr->alu.dst.n_bits,
10422                         instr->alu.dst.offset,
10423                         instr->alu.src_val);
10424 }
10425
10426 static void
10427 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10428 {
10429         int prefetch  = 0, idx_imm = 0, src_imm = 0;
10430
10431         if (instr->type == INSTR_REGPREFETCH_RH ||
10432             instr->type == INSTR_REGPREFETCH_RM ||
10433             instr->type == INSTR_REGPREFETCH_RI)
10434                 prefetch = 1;
10435
10436         /* index is the 3rd operand for the regrd instruction and the 2nd
10437          * operand for the regwr and regadd instructions.
10438          */
10439         if (instr->type == INSTR_REGPREFETCH_RI ||
10440             instr->type == INSTR_REGRD_HRI ||
10441             instr->type == INSTR_REGRD_MRI ||
10442             instr->type == INSTR_REGWR_RIH ||
10443             instr->type == INSTR_REGWR_RIM ||
10444             instr->type == INSTR_REGWR_RII ||
10445             instr->type == INSTR_REGADD_RIH ||
10446             instr->type == INSTR_REGADD_RIM ||
10447             instr->type == INSTR_REGADD_RII)
10448                 idx_imm = 1;
10449
10450         /* src is the 3rd operand for the regwr and regadd instructions. */
10451         if (instr->type == INSTR_REGWR_RHI ||
10452             instr->type == INSTR_REGWR_RMI ||
10453             instr->type == INSTR_REGWR_RII ||
10454             instr->type == INSTR_REGADD_RHI ||
10455             instr->type == INSTR_REGADD_RMI ||
10456             instr->type == INSTR_REGADD_RII)
10457                 src_imm = 1;
10458
10459         /* instr.regarray.regarray_id. */
10460         fprintf(f,
10461                 "\t{\n"
10462                 "\t\t.type = %s,\n"
10463                 "\t\t.regarray = {\n"
10464                 "\t\t\t.regarray_id = %u,\n",
10465                 instr_type_to_name(instr),
10466                 instr->regarray.regarray_id);
10467
10468         /* instr.regarray.idx / instr.regarray.idx_val. */
10469         if (!idx_imm)
10470                 fprintf(f,
10471                         "\t\t\t\t.idx = {\n"
10472                         "\t\t\t\t\t.struct_id = %u,\n"
10473                         "\t\t\t\t\t.n_bits = %u,\n"
10474                         "\t\t\t\t\t.offset = %u,\n"
10475                         "\t\t\t\t},\n",
10476                         instr->regarray.idx.struct_id,
10477                         instr->regarray.idx.n_bits,
10478                         instr->regarray.idx.offset);
10479         else
10480                 fprintf(f,
10481                         "\t\t\t\t.idx_val = %u,\n",
10482                         instr->regarray.idx_val);
10483
10484         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10485         if (!prefetch) {
10486                 if (!src_imm)
10487                         fprintf(f,
10488                                 "\t\t\t\t.dstsrc = {\n"
10489                                 "\t\t\t\t\t.struct_id = %u,\n"
10490                                 "\t\t\t\t\t.n_bits = %u,\n"
10491                                 "\t\t\t\t\t.offset = %u,\n"
10492                                 "\t\t\t\t},\n",
10493                                 instr->regarray.dstsrc.struct_id,
10494                                 instr->regarray.dstsrc.n_bits,
10495                                 instr->regarray.dstsrc.offset);
10496                 else
10497                         fprintf(f,
10498                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10499                                 instr->regarray.dstsrc_val);
10500         }
10501
10502         /* instr.regarray and instr - closing curly braces. */
10503         fprintf(f,
10504                 "\t\t},\n"
10505                 "\t},\n");
10506 }
10507
10508 static void
10509 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10510 {
10511         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
10512
10513         if (instr->type == INSTR_METPREFETCH_H ||
10514             instr->type == INSTR_METPREFETCH_M ||
10515             instr->type == INSTR_METPREFETCH_I)
10516                 prefetch = 1;
10517
10518         /* idx_imm. */
10519         if (instr->type == INSTR_METPREFETCH_I ||
10520             instr->type == INSTR_METER_IHM ||
10521             instr->type == INSTR_METER_IHI ||
10522             instr->type == INSTR_METER_IMM ||
10523             instr->type == INSTR_METER_IMI)
10524                 idx_imm = 1;
10525
10526         /* color_in_imm. */
10527         if (instr->type == INSTR_METER_HHI ||
10528             instr->type == INSTR_METER_HMI ||
10529             instr->type == INSTR_METER_MHI ||
10530             instr->type == INSTR_METER_MMI ||
10531             instr->type == INSTR_METER_IHI ||
10532             instr->type == INSTR_METER_IMI)
10533                 color_in_imm = 1;
10534
10535         /* instr.meter.metarray_id. */
10536         fprintf(f,
10537                 "\t{\n"
10538                 "\t\t.type = %s,\n"
10539                 "\t\t.meter = {\n"
10540                 "\t\t\t.metarray_id = %u,\n",
10541                 instr_type_to_name(instr),
10542                 instr->meter.metarray_id);
10543
10544         /* instr.meter.idx / instr.meter.idx_val. */
10545         if (!idx_imm)
10546                 fprintf(f,
10547                         "\t\t\t.idx = {\n"
10548                         "\t\t\t\t.struct_id = %u,\n"
10549                         "\t\t\t\t.n_bits = %u,\n"
10550                         "\t\t\t\t.offset = %u,\n"
10551                         "\t\t\t},\n",
10552                         instr->meter.idx.struct_id,
10553                         instr->meter.idx.n_bits,
10554                         instr->meter.idx.offset);
10555         else
10556                 fprintf(f,
10557                         "\t\t\t.idx_val = %u,\n",
10558                         instr->meter.idx_val);
10559
10560         if (!prefetch) {
10561                 /* instr.meter.length. */
10562                 fprintf(f,
10563                         "\t\t\t.length = {\n"
10564                         "\t\t\t\t.struct_id = %u,\n"
10565                         "\t\t\t\t.n_bits = %u,\n"
10566                         "\t\t\t\t.offset = %u,\n"
10567                         "\t\t\t},\n",
10568                         instr->meter.length.struct_id,
10569                         instr->meter.length.n_bits,
10570                         instr->meter.length.offset);
10571
10572                 /* instr.meter.color_in / instr.meter.color_in_val. */
10573                 if (!color_in_imm)
10574                         fprintf(f,
10575                                 "\t\t\t.color_in = {\n"
10576                                 "\t\t\t\t.struct_id = %u,\n"
10577                                 "\t\t\t\t.n_bits = %u,\n"
10578                                 "\t\t\t\t.offset = %u,\n"
10579                                 "\t\t\t},\n",
10580                                 instr->meter.color_in.struct_id,
10581                                 instr->meter.color_in.n_bits,
10582                                 instr->meter.color_in.offset);
10583                 else
10584                         fprintf(f,
10585                                 "\t\t\t.color_in_val = %u,\n",
10586                                 (uint32_t)instr->meter.color_in_val);
10587
10588                 /* instr.meter.color_out. */
10589                 fprintf(f,
10590                         "\t\t\t.color_out = {\n"
10591                         "\t\t\t\t.struct_id = %u,\n"
10592                         "\t\t\t\t.n_bits = %u,\n"
10593                         "\t\t\t\t.offset = %u,\n"
10594                         "\t\t\t},\n",
10595                         instr->meter.color_out.struct_id,
10596                         instr->meter.color_out.n_bits,
10597                         instr->meter.color_out.offset);
10598         }
10599
10600         /* instr.meter and instr - closing curly braces. */
10601         fprintf(f,
10602                 "\t\t},\n"
10603                 "\t},\n");
10604 }
10605
10606 static void
10607 instr_table_export(struct instruction *instr,
10608                 FILE *f)
10609 {
10610         fprintf(f,
10611                 "\t{\n"
10612                 "\t\t.type = %s,\n"
10613                 "\t\t.table = {\n"
10614                 "\t\t\t.table_id = %u,\n"
10615                 "\t\t},\n"
10616                 "\t},\n",
10617                 instr_type_to_name(instr),
10618                 instr->table.table_id);
10619 }
10620
10621 static void
10622 instr_learn_export(struct instruction *instr, FILE *f)
10623 {
10624         fprintf(f,
10625                 "\t{\n"
10626                 "\t\t.type = %s,\n"
10627                 "\t\t.learn = {\n"
10628                 "\t\t\t\t.action_id = %u,\n"
10629                 "\t\t},\n"
10630                 "\t},\n",
10631                 instr_type_to_name(instr),
10632                 instr->learn.action_id);
10633 }
10634
10635 static void
10636 instr_forget_export(struct instruction *instr, FILE *f)
10637 {
10638         fprintf(f,
10639                 "\t{\n"
10640                 "\t\t.type = %s,\n"
10641                 "\t},\n",
10642                 instr_type_to_name(instr));
10643 }
10644
10645 static void
10646 instr_extern_export(struct instruction *instr, FILE *f)
10647 {
10648         if (instr->type == INSTR_EXTERN_OBJ)
10649                 fprintf(f,
10650                         "\t{\n"
10651                         "\t\t.type = %s,\n"
10652                         "\t\t.ext_obj = {\n"
10653                         "\t\t\t.ext_obj_id = %u,\n"
10654                         "\t\t\t.func_id = %u,\n"
10655                         "\t\t},\n"
10656                         "\t},\n",
10657                         instr_type_to_name(instr),
10658                         instr->ext_obj.ext_obj_id,
10659                         instr->ext_obj.func_id);
10660         else
10661                 fprintf(f,
10662                         "\t{\n"
10663                         "\t\t.type = %s,\n"
10664                         "\t\t.ext_func = {\n"
10665                         "\t\t\t.ext_func_id = %u,\n"
10666                         "\t\t},\n"
10667                         "\t},\n",
10668                         instr_type_to_name(instr),
10669                         instr->ext_func.ext_func_id);
10670 }
10671
10672 static void
10673 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10674 {
10675         fprintf(f,
10676                 "\t{\n"
10677                 "\t\t.type = %s,\n"
10678                 "\t\t.jmp = {\n"
10679                 "\t\t\t.ip = NULL,\n",
10680                 instr_type_to_name(instr));
10681
10682         switch (instr->type) {
10683         case INSTR_JMP_VALID:
10684         case INSTR_JMP_INVALID:
10685                 fprintf(f,
10686                         "\t\t\t.header_id = %u,\n",
10687                         instr->jmp.header_id);
10688                 break;
10689
10690         case INSTR_JMP_ACTION_HIT:
10691         case INSTR_JMP_ACTION_MISS:
10692                 fprintf(f,
10693                         "\t\t\t.action_id = %u,\n",
10694                         instr->jmp.action_id);
10695                 break;
10696
10697         case INSTR_JMP_EQ:
10698         case INSTR_JMP_EQ_MH:
10699         case INSTR_JMP_EQ_HM:
10700         case INSTR_JMP_EQ_HH:
10701         case INSTR_JMP_NEQ:
10702         case INSTR_JMP_NEQ_MH:
10703         case INSTR_JMP_NEQ_HM:
10704         case INSTR_JMP_NEQ_HH:
10705         case INSTR_JMP_LT:
10706         case INSTR_JMP_LT_MH:
10707         case INSTR_JMP_LT_HM:
10708         case INSTR_JMP_LT_HH:
10709         case INSTR_JMP_GT:
10710         case INSTR_JMP_GT_MH:
10711         case INSTR_JMP_GT_HM:
10712         case INSTR_JMP_GT_HH:
10713                 fprintf(f,
10714                         "\t\t\t.a = {\n"
10715                         "\t\t\t\t.struct_id = %u,\n"
10716                         "\t\t\t\t.n_bits = %u,\n"
10717                         "\t\t\t\t.offset = %u,\n"
10718                         "\t\t\t},\n"
10719                         "\t\t\t.b = {\n"
10720                         "\t\t\t\t.struct_id = %u,\n"
10721                         "\t\t\t\t.n_bits = %u,\n"
10722                         "\t\t\t\t.offset = %u,\n"
10723                         "\t\t\t},\n",
10724                         instr->jmp.a.struct_id,
10725                         instr->jmp.a.n_bits,
10726                         instr->jmp.a.offset,
10727                         instr->jmp.b.struct_id,
10728                         instr->jmp.b.n_bits,
10729                         instr->jmp.b.offset);
10730                 break;
10731
10732         case INSTR_JMP_EQ_I:
10733         case INSTR_JMP_NEQ_I:
10734         case INSTR_JMP_LT_MI:
10735         case INSTR_JMP_LT_HI:
10736         case INSTR_JMP_GT_MI:
10737         case INSTR_JMP_GT_HI:
10738                 fprintf(f,
10739                         "\t\t\t.a = {\n"
10740                         "\t\t\t\t.struct_id = %u,\n"
10741                         "\t\t\t\t.n_bits = %u,\n"
10742                         "\t\t\t\t.offset = %u,\n"
10743                         "\t\t\t}\n,"
10744                         "\t\t\t.b_val = %" PRIu64 ",\n",
10745                         instr->jmp.a.struct_id,
10746                         instr->jmp.a.n_bits,
10747                         instr->jmp.a.offset,
10748                         instr->jmp.b_val);
10749                 break;
10750
10751         default:
10752                 break;
10753         }
10754
10755         fprintf(f,
10756                 "\t\t},\n"
10757                 "\t},\n");
10758 }
10759
10760 static void
10761 instr_return_export(struct instruction *instr,
10762                 FILE *f)
10763 {
10764         fprintf(f,
10765                 "\t{\n"
10766                 "\t\t.type = %s,\n",
10767                 instr_type_to_name(instr));
10768
10769         fprintf(f,
10770                 "\t},\n");
10771 }
10772
10773 static instruction_export_t export_table[] = {
10774         [INSTR_RX] = instr_io_export,
10775
10776         [INSTR_TX] = instr_io_export,
10777         [INSTR_TX_I] = instr_io_export,
10778
10779         [INSTR_HDR_EXTRACT] = instr_io_export,
10780         [INSTR_HDR_EXTRACT2] = instr_io_export,
10781         [INSTR_HDR_EXTRACT3] = instr_io_export,
10782         [INSTR_HDR_EXTRACT4] = instr_io_export,
10783         [INSTR_HDR_EXTRACT5] = instr_io_export,
10784         [INSTR_HDR_EXTRACT6] = instr_io_export,
10785         [INSTR_HDR_EXTRACT7] = instr_io_export,
10786         [INSTR_HDR_EXTRACT8] = instr_io_export,
10787
10788         [INSTR_HDR_EXTRACT_M] = instr_io_export,
10789
10790         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10791
10792         [INSTR_HDR_EMIT] = instr_io_export,
10793         [INSTR_HDR_EMIT_TX] = instr_io_export,
10794         [INSTR_HDR_EMIT2_TX] = instr_io_export,
10795         [INSTR_HDR_EMIT3_TX] = instr_io_export,
10796         [INSTR_HDR_EMIT4_TX] = instr_io_export,
10797         [INSTR_HDR_EMIT5_TX] = instr_io_export,
10798         [INSTR_HDR_EMIT6_TX] = instr_io_export,
10799         [INSTR_HDR_EMIT7_TX] = instr_io_export,
10800         [INSTR_HDR_EMIT8_TX] = instr_io_export,
10801
10802         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10803         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10804
10805         [INSTR_MOV] = instr_mov_export,
10806         [INSTR_MOV_MH] = instr_mov_export,
10807         [INSTR_MOV_HM] = instr_mov_export,
10808         [INSTR_MOV_HH] = instr_mov_export,
10809         [INSTR_MOV_I] = instr_mov_export,
10810
10811         [INSTR_DMA_HT]  = instr_dma_ht_export,
10812         [INSTR_DMA_HT2] = instr_dma_ht_export,
10813         [INSTR_DMA_HT3] = instr_dma_ht_export,
10814         [INSTR_DMA_HT4] = instr_dma_ht_export,
10815         [INSTR_DMA_HT5] = instr_dma_ht_export,
10816         [INSTR_DMA_HT6] = instr_dma_ht_export,
10817         [INSTR_DMA_HT7] = instr_dma_ht_export,
10818         [INSTR_DMA_HT8] = instr_dma_ht_export,
10819
10820         [INSTR_ALU_ADD] = instr_alu_export,
10821         [INSTR_ALU_ADD_MH] = instr_alu_export,
10822         [INSTR_ALU_ADD_HM] = instr_alu_export,
10823         [INSTR_ALU_ADD_HH] = instr_alu_export,
10824         [INSTR_ALU_ADD_MI] = instr_alu_export,
10825         [INSTR_ALU_ADD_HI] = instr_alu_export,
10826
10827         [INSTR_ALU_SUB] = instr_alu_export,
10828         [INSTR_ALU_SUB_MH] = instr_alu_export,
10829         [INSTR_ALU_SUB_HM] = instr_alu_export,
10830         [INSTR_ALU_SUB_HH] = instr_alu_export,
10831         [INSTR_ALU_SUB_MI] = instr_alu_export,
10832         [INSTR_ALU_SUB_HI] = instr_alu_export,
10833
10834         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10835         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10836         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10837         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10838
10839         [INSTR_ALU_AND] = instr_alu_export,
10840         [INSTR_ALU_AND_MH] = instr_alu_export,
10841         [INSTR_ALU_AND_HM] = instr_alu_export,
10842         [INSTR_ALU_AND_HH] = instr_alu_export,
10843         [INSTR_ALU_AND_I] = instr_alu_export,
10844
10845         [INSTR_ALU_OR] = instr_alu_export,
10846         [INSTR_ALU_OR_MH] = instr_alu_export,
10847         [INSTR_ALU_OR_HM] = instr_alu_export,
10848         [INSTR_ALU_OR_HH] = instr_alu_export,
10849         [INSTR_ALU_OR_I] = instr_alu_export,
10850
10851         [INSTR_ALU_XOR] = instr_alu_export,
10852         [INSTR_ALU_XOR_MH] = instr_alu_export,
10853         [INSTR_ALU_XOR_HM] = instr_alu_export,
10854         [INSTR_ALU_XOR_HH] = instr_alu_export,
10855         [INSTR_ALU_XOR_I] = instr_alu_export,
10856
10857         [INSTR_ALU_SHL] = instr_alu_export,
10858         [INSTR_ALU_SHL_MH] = instr_alu_export,
10859         [INSTR_ALU_SHL_HM] = instr_alu_export,
10860         [INSTR_ALU_SHL_HH] = instr_alu_export,
10861         [INSTR_ALU_SHL_MI] = instr_alu_export,
10862         [INSTR_ALU_SHL_HI] = instr_alu_export,
10863
10864         [INSTR_ALU_SHR] = instr_alu_export,
10865         [INSTR_ALU_SHR_MH] = instr_alu_export,
10866         [INSTR_ALU_SHR_HM] = instr_alu_export,
10867         [INSTR_ALU_SHR_HH] = instr_alu_export,
10868         [INSTR_ALU_SHR_MI] = instr_alu_export,
10869         [INSTR_ALU_SHR_HI] = instr_alu_export,
10870
10871         [INSTR_REGPREFETCH_RH] = instr_reg_export,
10872         [INSTR_REGPREFETCH_RM] = instr_reg_export,
10873         [INSTR_REGPREFETCH_RI] = instr_reg_export,
10874
10875         [INSTR_REGRD_HRH] = instr_reg_export,
10876         [INSTR_REGRD_HRM] = instr_reg_export,
10877         [INSTR_REGRD_MRH] = instr_reg_export,
10878         [INSTR_REGRD_MRM] = instr_reg_export,
10879         [INSTR_REGRD_HRI] = instr_reg_export,
10880         [INSTR_REGRD_MRI] = instr_reg_export,
10881
10882         [INSTR_REGWR_RHH] = instr_reg_export,
10883         [INSTR_REGWR_RHM] = instr_reg_export,
10884         [INSTR_REGWR_RMH] = instr_reg_export,
10885         [INSTR_REGWR_RMM] = instr_reg_export,
10886         [INSTR_REGWR_RHI] = instr_reg_export,
10887         [INSTR_REGWR_RMI] = instr_reg_export,
10888         [INSTR_REGWR_RIH] = instr_reg_export,
10889         [INSTR_REGWR_RIM] = instr_reg_export,
10890         [INSTR_REGWR_RII] = instr_reg_export,
10891
10892         [INSTR_REGADD_RHH] = instr_reg_export,
10893         [INSTR_REGADD_RHM] = instr_reg_export,
10894         [INSTR_REGADD_RMH] = instr_reg_export,
10895         [INSTR_REGADD_RMM] = instr_reg_export,
10896         [INSTR_REGADD_RHI] = instr_reg_export,
10897         [INSTR_REGADD_RMI] = instr_reg_export,
10898         [INSTR_REGADD_RIH] = instr_reg_export,
10899         [INSTR_REGADD_RIM] = instr_reg_export,
10900         [INSTR_REGADD_RII] = instr_reg_export,
10901
10902         [INSTR_METPREFETCH_H] = instr_meter_export,
10903         [INSTR_METPREFETCH_M] = instr_meter_export,
10904         [INSTR_METPREFETCH_I] = instr_meter_export,
10905
10906         [INSTR_METER_HHM] = instr_meter_export,
10907         [INSTR_METER_HHI] = instr_meter_export,
10908         [INSTR_METER_HMM] = instr_meter_export,
10909         [INSTR_METER_HMI] = instr_meter_export,
10910         [INSTR_METER_MHM] = instr_meter_export,
10911         [INSTR_METER_MHI] = instr_meter_export,
10912         [INSTR_METER_MMM] = instr_meter_export,
10913         [INSTR_METER_MMI] = instr_meter_export,
10914         [INSTR_METER_IHM] = instr_meter_export,
10915         [INSTR_METER_IHI] = instr_meter_export,
10916         [INSTR_METER_IMM] = instr_meter_export,
10917         [INSTR_METER_IMI] = instr_meter_export,
10918
10919         [INSTR_TABLE] = instr_table_export,
10920         [INSTR_TABLE_AF] = instr_table_export,
10921         [INSTR_SELECTOR] = instr_table_export,
10922         [INSTR_LEARNER] = instr_table_export,
10923         [INSTR_LEARNER_AF] = instr_table_export,
10924
10925         [INSTR_LEARNER_LEARN] = instr_learn_export,
10926         [INSTR_LEARNER_FORGET] = instr_forget_export,
10927
10928         [INSTR_EXTERN_OBJ] = instr_extern_export,
10929         [INSTR_EXTERN_FUNC] = instr_extern_export,
10930
10931         [INSTR_JMP] = instr_jmp_export,
10932         [INSTR_JMP_VALID] = instr_jmp_export,
10933         [INSTR_JMP_INVALID] = instr_jmp_export,
10934         [INSTR_JMP_HIT] = instr_jmp_export,
10935         [INSTR_JMP_MISS] = instr_jmp_export,
10936         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
10937         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
10938
10939         [INSTR_JMP_EQ] = instr_jmp_export,
10940         [INSTR_JMP_EQ_MH] = instr_jmp_export,
10941         [INSTR_JMP_EQ_HM] = instr_jmp_export,
10942         [INSTR_JMP_EQ_HH] = instr_jmp_export,
10943         [INSTR_JMP_EQ_I] = instr_jmp_export,
10944
10945         [INSTR_JMP_NEQ] = instr_jmp_export,
10946         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
10947         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
10948         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
10949         [INSTR_JMP_NEQ_I] = instr_jmp_export,
10950
10951         [INSTR_JMP_LT] = instr_jmp_export,
10952         [INSTR_JMP_LT_MH] = instr_jmp_export,
10953         [INSTR_JMP_LT_HM] = instr_jmp_export,
10954         [INSTR_JMP_LT_HH] = instr_jmp_export,
10955         [INSTR_JMP_LT_MI] = instr_jmp_export,
10956         [INSTR_JMP_LT_HI] = instr_jmp_export,
10957
10958         [INSTR_JMP_GT] = instr_jmp_export,
10959         [INSTR_JMP_GT_MH] = instr_jmp_export,
10960         [INSTR_JMP_GT_HM] = instr_jmp_export,
10961         [INSTR_JMP_GT_HH] = instr_jmp_export,
10962         [INSTR_JMP_GT_MI] = instr_jmp_export,
10963         [INSTR_JMP_GT_HI] = instr_jmp_export,
10964
10965         [INSTR_RETURN] = instr_return_export,
10966 };
10967
10968 static void
10969 action_data_codegen(struct action *a, FILE *f)
10970 {
10971         uint32_t i;
10972
10973         fprintf(f,
10974                 "static const struct instruction action_%s_instructions[] = {\n",
10975                 a->name);
10976
10977         for (i = 0; i < a->n_instructions; i++) {
10978                 struct instruction *instr = &a->instructions[i];
10979                 instruction_export_t func = export_table[instr->type];
10980
10981                 func(instr, f);
10982         }
10983
10984         fprintf(f, "};\n");
10985 }
10986
10987 static const char *
10988 instr_type_to_func(struct instruction *instr)
10989 {
10990         switch (instr->type) {
10991         case INSTR_RX: return NULL;
10992
10993         case INSTR_TX: return "__instr_tx_exec";
10994         case INSTR_TX_I: return "__instr_tx_i_exec";
10995
10996         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
10997         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
10998         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
10999         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11000         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11001         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11002         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11003         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11004
11005         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11006
11007         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11008
11009         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11010         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11011         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11012         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11013         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11014         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11015         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11016         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11017         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11018
11019         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11020         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11021
11022         case INSTR_MOV: return "__instr_mov_exec";
11023         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11024         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11025         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11026         case INSTR_MOV_I: return "__instr_mov_i_exec";
11027
11028         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11029         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11030         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11031         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11032         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11033         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11034         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11035         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11036
11037         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11038         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11039         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11040         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11041         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11042         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11043
11044         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11045         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11046         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11047         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11048         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11049         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11050
11051         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11052         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11053         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11054         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11055
11056         case INSTR_ALU_AND: return "__instr_alu_and_exec";
11057         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11058         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11059         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11060         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11061
11062         case INSTR_ALU_OR: return "__instr_alu_or_exec";
11063         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11064         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11065         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11066         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11067
11068         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11069         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11070         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11071         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11072         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11073
11074         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11075         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11076         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11077         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11078         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11079         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11080
11081         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11082         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11083         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11084         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11085         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11086         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11087
11088         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11089         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11090         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11091
11092         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11093         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11094         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11095         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11096         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11097         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11098
11099         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11100         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11101         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11102         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11103         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11104         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11105         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11106         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11107         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11108
11109         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11110         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11111         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11112         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11113         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11114         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11115         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11116         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11117         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11118
11119         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11120         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11121         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11122
11123         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11124         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11125         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11126         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11127         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11128         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11129         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11130         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11131         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11132         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11133         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11134         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11135
11136         case INSTR_TABLE: return NULL;
11137         case INSTR_TABLE_AF: return NULL;
11138         case INSTR_SELECTOR: return NULL;
11139         case INSTR_LEARNER: return NULL;
11140         case INSTR_LEARNER_AF: return NULL;
11141
11142         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11143         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11144
11145         case INSTR_EXTERN_OBJ: return NULL;
11146         case INSTR_EXTERN_FUNC: return NULL;
11147
11148         case INSTR_JMP: return NULL;
11149         case INSTR_JMP_VALID: return NULL;
11150         case INSTR_JMP_INVALID: return NULL;
11151         case INSTR_JMP_HIT: return NULL;
11152         case INSTR_JMP_MISS: return NULL;
11153         case INSTR_JMP_ACTION_HIT: return NULL;
11154         case INSTR_JMP_ACTION_MISS: return NULL;
11155         case INSTR_JMP_EQ: return NULL;
11156         case INSTR_JMP_EQ_MH: return NULL;
11157         case INSTR_JMP_EQ_HM: return NULL;
11158         case INSTR_JMP_EQ_HH: return NULL;
11159         case INSTR_JMP_EQ_I: return NULL;
11160         case INSTR_JMP_NEQ: return NULL;
11161         case INSTR_JMP_NEQ_MH: return NULL;
11162         case INSTR_JMP_NEQ_HM: return NULL;
11163         case INSTR_JMP_NEQ_HH: return NULL;
11164         case INSTR_JMP_NEQ_I: return NULL;
11165         case INSTR_JMP_LT: return NULL;
11166         case INSTR_JMP_LT_MH: return NULL;
11167         case INSTR_JMP_LT_HM: return NULL;
11168         case INSTR_JMP_LT_HH: return NULL;
11169         case INSTR_JMP_LT_MI: return NULL;
11170         case INSTR_JMP_LT_HI: return NULL;
11171         case INSTR_JMP_GT: return NULL;
11172         case INSTR_JMP_GT_MH: return NULL;
11173         case INSTR_JMP_GT_HM: return NULL;
11174         case INSTR_JMP_GT_HH: return NULL;
11175         case INSTR_JMP_GT_MI: return NULL;
11176         case INSTR_JMP_GT_HI: return NULL;
11177
11178         case INSTR_RETURN: return NULL;
11179
11180         default: return NULL;
11181         }
11182 }
11183
11184 static void
11185 action_instr_does_tx_codegen(struct action *a,
11186                         uint32_t instr_pos,
11187                         struct instruction *instr,
11188                         FILE *f)
11189 {
11190         fprintf(f,
11191                 "%s(p, t, &action_%s_instructions[%u]);\n"
11192                 "\tthread_ip_reset(p, t);\n"
11193                 "\tinstr_rx_exec(p);\n"
11194                 "\treturn;\n",
11195                 instr_type_to_func(instr),
11196                 a->name,
11197                 instr_pos);
11198 }
11199
11200 static void
11201 action_instr_extern_obj_codegen(struct action *a,
11202                                 uint32_t instr_pos,
11203                                 FILE *f)
11204 {
11205         fprintf(f,
11206                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11207                 a->name,
11208                 instr_pos);
11209 }
11210
11211 static void
11212 action_instr_extern_func_codegen(struct action *a,
11213                                  uint32_t instr_pos,
11214                                  FILE *f)
11215 {
11216         fprintf(f,
11217                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11218                 a->name,
11219                 instr_pos);
11220 }
11221
11222 static void
11223 action_instr_jmp_codegen(struct action *a,
11224                          uint32_t instr_pos,
11225                          struct instruction *instr,
11226                          struct instruction_data *data,
11227                          FILE *f)
11228 {
11229         switch (instr->type) {
11230         case INSTR_JMP:
11231                 fprintf(f,
11232                         "goto %s;\n",
11233                         data->jmp_label);
11234                 return;
11235
11236         case INSTR_JMP_VALID:
11237                 fprintf(f,
11238                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11239                         "\t\tgoto %s;\n",
11240                         a->name,
11241                         instr_pos,
11242                         data->jmp_label);
11243                 return;
11244
11245         case INSTR_JMP_INVALID:
11246                 fprintf(f,
11247                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11248                         "\t\tgoto %s;\n",
11249                         a->name,
11250                         instr_pos,
11251                         data->jmp_label);
11252                 return;
11253
11254         case INSTR_JMP_HIT:
11255                 fprintf(f,
11256                         "if (t->hit)\n"
11257                         "\t\tgoto %s;\n",
11258                         data->jmp_label);
11259                 return;
11260
11261         case INSTR_JMP_MISS:
11262                 fprintf(f,
11263                         "if (!t->hit)\n"
11264                         "\t\tgoto %s;\n",
11265                         data->jmp_label);
11266                 return;
11267
11268         case INSTR_JMP_ACTION_HIT:
11269                 fprintf(f,
11270                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11271                         "\t\tgoto %s;\n",
11272                         a->name,
11273                         instr_pos,
11274                         data->jmp_label);
11275                 return;
11276
11277         case INSTR_JMP_ACTION_MISS:
11278                 fprintf(f,
11279                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11280                         "\t\tgoto %s;\n",
11281                         a->name,
11282                         instr_pos,
11283                         data->jmp_label);
11284                 return;
11285
11286         case INSTR_JMP_EQ:
11287                 fprintf(f,
11288                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11289                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11290                         "\t\tgoto %s;\n",
11291                         a->name,
11292                         instr_pos,
11293                         a->name,
11294                         instr_pos,
11295                         data->jmp_label);
11296                 return;
11297
11298         case INSTR_JMP_EQ_MH:
11299                 fprintf(f,
11300                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11301                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11302                         "\t\tgoto %s;\n",
11303                         a->name,
11304                         instr_pos,
11305                         a->name,
11306                         instr_pos,
11307                         data->jmp_label);
11308                 return;
11309
11310         case INSTR_JMP_EQ_HM:
11311                 fprintf(f,
11312                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11313                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11314                         "\t\tgoto %s;\n",
11315                         a->name,
11316                         instr_pos,
11317                         a->name,
11318                         instr_pos,
11319                         data->jmp_label);
11320                 return;
11321
11322         case INSTR_JMP_EQ_HH:
11323                 fprintf(f,
11324                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11325                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11326                         "\t\tgoto %s;\n",
11327                         a->name,
11328                         instr_pos,
11329                         a->name,
11330                         instr_pos,
11331                         data->jmp_label);
11332                 return;
11333
11334         case INSTR_JMP_EQ_I:
11335                 fprintf(f,
11336                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11337                         "action_%s_instructions[%u].jmp.b_val)\n"
11338                         "\t\tgoto %s;\n",
11339                         a->name,
11340                         instr_pos,
11341                         a->name,
11342                         instr_pos,
11343                         data->jmp_label);
11344                 return;
11345
11346         case INSTR_JMP_NEQ:
11347                 fprintf(f,
11348                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11349                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11350                         "\t\tgoto %s;\n",
11351                         a->name,
11352                         instr_pos,
11353                         a->name,
11354                         instr_pos,
11355                         data->jmp_label);
11356                 return;
11357
11358         case INSTR_JMP_NEQ_MH:
11359                 fprintf(f,
11360                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11361                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11362                         "\t\tgoto %s;\n",
11363                         a->name,
11364                         instr_pos,
11365                         a->name,
11366                         instr_pos,
11367                         data->jmp_label);
11368                 return;
11369
11370         case INSTR_JMP_NEQ_HM:
11371                 fprintf(f,
11372                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11373                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11374                         "\t\tgoto %s;\n",
11375                         a->name,
11376                         instr_pos,
11377                         a->name,
11378                         instr_pos,
11379                         data->jmp_label);
11380                 return;
11381
11382         case INSTR_JMP_NEQ_HH:
11383                 fprintf(f,
11384                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11385                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11386                         "\t\tgoto %s;\n",
11387                         a->name,
11388                         instr_pos,
11389                         a->name,
11390                         instr_pos,
11391                         data->jmp_label);
11392                 return;
11393
11394         case INSTR_JMP_NEQ_I:
11395                 fprintf(f,
11396                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11397                         "action_%s_instructions[%u].jmp.b_val)\n"
11398                         "\t\tgoto %s;\n",
11399                         a->name,
11400                         instr_pos,
11401                         a->name,
11402                         instr_pos,
11403                         data->jmp_label);
11404                 return;
11405
11406         case INSTR_JMP_LT:
11407                 fprintf(f,
11408                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11409                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11410                         "\t\tgoto %s;\n",
11411                         a->name,
11412                         instr_pos,
11413                         a->name,
11414                         instr_pos,
11415                         data->jmp_label);
11416                 return;
11417
11418         case INSTR_JMP_LT_MH:
11419                 fprintf(f,
11420                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11421                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11422                         "\t\tgoto %s;\n",
11423                         a->name,
11424                         instr_pos,
11425                         a->name,
11426                         instr_pos,
11427                         data->jmp_label);
11428                 return;
11429
11430         case INSTR_JMP_LT_HM:
11431                 fprintf(f,
11432                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11433                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11434                         "\t\tgoto %s;\n",
11435                         a->name,
11436                         instr_pos,
11437                         a->name,
11438                         instr_pos,
11439                         data->jmp_label);
11440                 return;
11441
11442         case INSTR_JMP_LT_HH:
11443                 fprintf(f,
11444                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11445                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11446                         "\t\tgoto %s;\n",
11447                         a->name,
11448                         instr_pos,
11449                         a->name,
11450                         instr_pos,
11451                         data->jmp_label);
11452                 return;
11453
11454         case INSTR_JMP_LT_MI:
11455                 fprintf(f,
11456                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11457                         "action_%s_instructions[%u].jmp.b_val)\n"
11458                         "\t\tgoto %s;\n",
11459                         a->name,
11460                         instr_pos,
11461                         a->name,
11462                         instr_pos,
11463                         data->jmp_label);
11464                 return;
11465
11466         case INSTR_JMP_LT_HI:
11467                 fprintf(f,
11468                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11469                         "action_%s_instructions[%u].jmp.b_val)\n"
11470                         "\t\tgoto %s;\n",
11471                         a->name,
11472                         instr_pos,
11473                         a->name,
11474                         instr_pos,
11475                         data->jmp_label);
11476                 return;
11477
11478         case INSTR_JMP_GT:
11479                 fprintf(f,
11480                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11481                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11482                         "\t\tgoto %s;\n",
11483                         a->name,
11484                         instr_pos,
11485                         a->name,
11486                         instr_pos,
11487                         data->jmp_label);
11488                 return;
11489
11490         case INSTR_JMP_GT_MH:
11491                 fprintf(f,
11492                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11493                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11494                         "\t\tgoto %s;\n",
11495                         a->name,
11496                         instr_pos,
11497                         a->name,
11498                         instr_pos,
11499                         data->jmp_label);
11500                 return;
11501
11502         case INSTR_JMP_GT_HM:
11503                 fprintf(f,
11504                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11505                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11506                         "\t\tgoto %s;\n",
11507                         a->name,
11508                         instr_pos,
11509                         a->name,
11510                         instr_pos,
11511                         data->jmp_label);
11512                 return;
11513
11514         case INSTR_JMP_GT_HH:
11515                 fprintf(f,
11516                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11517                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11518                         "\t\tgoto %s;\n",
11519                         a->name,
11520                         instr_pos,
11521                         a->name,
11522                         instr_pos,
11523                         data->jmp_label);
11524                 return;
11525
11526         case INSTR_JMP_GT_MI:
11527                 fprintf(f,
11528                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11529                         "action_%s_instructions[%u].jmp.b_val)\n"
11530                         "\t\tgoto %s;\n",
11531                         a->name,
11532                         instr_pos,
11533                         a->name,
11534                         instr_pos,
11535                         data->jmp_label);
11536                 return;
11537
11538         case INSTR_JMP_GT_HI:
11539                 fprintf(f,
11540                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11541                         "action_%s_instructions[%u].jmp.b_val)\n"
11542                         "\t\tgoto %s;\n",
11543                         a->name,
11544                         instr_pos,
11545                         a->name,
11546                         instr_pos,
11547                         data->jmp_label);
11548                 return;
11549
11550         default:
11551                 return;
11552         }
11553 }
11554
11555 static void
11556 action_instr_return_codegen(FILE *f)
11557 {
11558         fprintf(f,
11559                 "return;\n");
11560 }
11561
11562 static void
11563 action_instr_codegen(struct action *a, FILE *f)
11564 {
11565         uint32_t i;
11566
11567         fprintf(f,
11568                 "void\n"
11569                 "action_%s_run(struct rte_swx_pipeline *p)\n"
11570                 "{\n"
11571                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11572                 "\n",
11573                 a->name);
11574
11575         for (i = 0; i < a->n_instructions; i++) {
11576                 struct instruction *instr = &a->instructions[i];
11577                 struct instruction_data *data = &a->instruction_data[i];
11578
11579                 /* Label, if present. */
11580                 if (data->label[0])
11581                         fprintf(f, "\n%s : ", data->label);
11582                 else
11583                         fprintf(f, "\n\t");
11584
11585                 /* TX instruction type. */
11586                 if (instruction_does_tx(instr)) {
11587                         action_instr_does_tx_codegen(a, i, instr, f);
11588                         continue;
11589                 }
11590
11591                 /* Extern object/function instruction type. */
11592                 if (instr->type == INSTR_EXTERN_OBJ) {
11593                         action_instr_extern_obj_codegen(a, i, f);
11594                         continue;
11595                 }
11596
11597                 if (instr->type == INSTR_EXTERN_FUNC) {
11598                         action_instr_extern_func_codegen(a, i, f);
11599                         continue;
11600                 }
11601
11602                 /* Jump instruction type. */
11603                 if (instruction_is_jmp(instr)) {
11604                         action_instr_jmp_codegen(a, i, instr, data, f);
11605                         continue;
11606                 }
11607
11608                 /* Return instruction type. */
11609                 if (instr->type == INSTR_RETURN) {
11610                         action_instr_return_codegen(f);
11611                         continue;
11612                 }
11613
11614                 /* Any other instruction type. */
11615                 fprintf(f,
11616                         "%s(p, t, &action_%s_instructions[%u]);\n",
11617                         instr_type_to_func(instr),
11618                         a->name,
11619                         i);
11620         }
11621
11622         fprintf(f, "}\n\n");
11623 }
11624
11625 struct instruction_group {
11626         TAILQ_ENTRY(instruction_group) node;
11627
11628         uint32_t group_id;
11629
11630         uint32_t first_instr_id;
11631
11632         uint32_t last_instr_id;
11633
11634         instr_exec_t func;
11635 };
11636
11637 TAILQ_HEAD(instruction_group_list, instruction_group);
11638
11639 static struct instruction_group *
11640 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11641 {
11642         struct instruction_group *g;
11643
11644         TAILQ_FOREACH(g, igl, node)
11645                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11646                         return g;
11647
11648         return NULL;
11649 }
11650
11651 static void
11652 instruction_group_list_free(struct instruction_group_list *igl)
11653 {
11654         if (!igl)
11655                 return;
11656
11657         for ( ; ; ) {
11658                 struct instruction_group *g;
11659
11660                 g = TAILQ_FIRST(igl);
11661                 if (!g)
11662                         break;
11663
11664                 TAILQ_REMOVE(igl, g, node);
11665                 free(g);
11666         }
11667
11668         free(igl);
11669 }
11670
11671 static struct instruction_group_list *
11672 instruction_group_list_create(struct rte_swx_pipeline *p)
11673 {
11674         struct instruction_group_list *igl = NULL;
11675         struct instruction_group *g = NULL;
11676         uint32_t n_groups = 0, i;
11677
11678         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11679                 goto error;
11680
11681         /* List init. */
11682         igl = calloc(1, sizeof(struct instruction_group_list));
11683         if (!igl)
11684                 goto error;
11685
11686         TAILQ_INIT(igl);
11687
11688         /* Allocate the first group. */
11689         g = calloc(1, sizeof(struct instruction_group));
11690         if (!g)
11691                 goto error;
11692
11693         /* Iteration 1: Separate the instructions into groups based on the thread yield
11694          * instructions. Do not worry about the jump instructions at this point.
11695          */
11696         for (i = 0; i < p->n_instructions; i++) {
11697                 struct instruction *instr = &p->instructions[i];
11698
11699                 /* Check for thread yield instructions. */
11700                 if (!instruction_does_thread_yield(instr))
11701                         continue;
11702
11703                 /* If the current group contains at least one instruction, then finalize it (with
11704                  * the previous instruction), add it to the list and allocate a new group (that
11705                  * starts with the current instruction).
11706                  */
11707                 if (i - g->first_instr_id) {
11708                         /* Finalize the group. */
11709                         g->last_instr_id = i - 1;
11710
11711                         /* Add the group to the list. Advance the number of groups. */
11712                         TAILQ_INSERT_TAIL(igl, g, node);
11713                         n_groups++;
11714
11715                         /* Allocate a new group. */
11716                         g = calloc(1, sizeof(struct instruction_group));
11717                         if (!g)
11718                                 goto error;
11719
11720                         /* Initialize the new group. */
11721                         g->group_id = n_groups;
11722                         g->first_instr_id = i;
11723                 }
11724
11725                 /* Finalize the current group (with the current instruction, therefore this group
11726                  * contains just the current thread yield instruction), add it to the list and
11727                  * allocate a new group (that starts with the next instruction).
11728                  */
11729
11730                 /* Finalize the group. */
11731                 g->last_instr_id = i;
11732
11733                 /* Add the group to the list. Advance the number of groups. */
11734                 TAILQ_INSERT_TAIL(igl, g, node);
11735                 n_groups++;
11736
11737                 /* Allocate a new group. */
11738                 g = calloc(1, sizeof(struct instruction_group));
11739                 if (!g)
11740                         goto error;
11741
11742                 /* Initialize the new group. */
11743                 g->group_id = n_groups;
11744                 g->first_instr_id = i + 1;
11745         }
11746
11747         /* Handle the last group. */
11748         if (i - g->first_instr_id) {
11749                 /* Finalize the group. */
11750                 g->last_instr_id = i - 1;
11751
11752                 /* Add the group to the list. Advance the number of groups. */
11753                 TAILQ_INSERT_TAIL(igl, g, node);
11754                 n_groups++;
11755         } else
11756                 free(g);
11757
11758         g = NULL;
11759
11760         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11761          * the destination of a jump instruction located in a different group ("far jump"), then the
11762          * current group has to be split, so that the instruction representing the far jump
11763          * destination is at the start of its group.
11764          */
11765         for ( ; ; ) {
11766                 int is_modified = 0;
11767
11768                 for (i = 0; i < p->n_instructions; i++) {
11769                         struct instruction_data *data = &p->instruction_data[i];
11770                         struct instruction_group *g;
11771                         uint32_t j;
11772
11773                         /* Continue when the current instruction is not a jump destination. */
11774                         if (!data->n_users)
11775                                 continue;
11776
11777                         g = instruction_group_list_group_find(igl, i);
11778                         if (!g)
11779                                 goto error;
11780
11781                         /* Find out all the jump instructions with this destination. */
11782                         for (j = 0; j < p->n_instructions; j++) {
11783                                 struct instruction *jmp_instr = &p->instructions[j];
11784                                 struct instruction_data *jmp_data = &p->instruction_data[j];
11785                                 struct instruction_group *jmp_g, *new_g;
11786
11787                                 /* Continue when not a jump instruction. Even when jump instruction,
11788                                  * continue when the jump destination is not this instruction.
11789                                  */
11790                                 if (!instruction_is_jmp(jmp_instr) ||
11791                                     strcmp(jmp_data->jmp_label, data->label))
11792                                         continue;
11793
11794                                 jmp_g = instruction_group_list_group_find(igl, j);
11795                                 if (!jmp_g)
11796                                         goto error;
11797
11798                                 /* Continue when both the jump instruction and the jump destination
11799                                  * instruction are in the same group. Even when in different groups,
11800                                  * still continue if the jump destination instruction is already the
11801                                  * first instruction of its group.
11802                                  */
11803                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11804                                         continue;
11805
11806                                 /* Split the group of the current jump destination instruction to
11807                                  * make this instruction the first instruction of a new group.
11808                                  */
11809                                 new_g = calloc(1, sizeof(struct instruction_group));
11810                                 if (!new_g)
11811                                         goto error;
11812
11813                                 new_g->group_id = n_groups;
11814                                 new_g->first_instr_id = i;
11815                                 new_g->last_instr_id = g->last_instr_id;
11816
11817                                 g->last_instr_id = i - 1;
11818
11819                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11820                                 n_groups++;
11821                                 is_modified = 1;
11822
11823                                 /* The decision to split this group (to make the current instruction
11824                                  * the first instruction of a new group) is already taken and fully
11825                                  * implemented, so no need to search for more reasons to do it.
11826                                  */
11827                                 break;
11828                         }
11829                 }
11830
11831                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11832                  * previously considered local (i.e. the jump destination is in the same group as
11833                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11834                  * different group than the jump instruction). Wost case scenario: each instruction
11835                  * that is a jump destination ends up as the first instruction of its group.
11836                  */
11837                 if (!is_modified)
11838                         break;
11839         }
11840
11841         /* Re-assign the group IDs to be in incremental order. */
11842         i = 0;
11843         TAILQ_FOREACH(g, igl, node) {
11844                 g->group_id = i;
11845
11846                 i++;
11847         }
11848
11849         return igl;
11850
11851 error:
11852         instruction_group_list_free(igl);
11853
11854         free(g);
11855
11856         return NULL;
11857 }
11858
11859 static void
11860 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11861                                uint32_t instr_pos,
11862                                struct instruction *instr,
11863                                FILE *f)
11864 {
11865         fprintf(f,
11866                 "%s(p, t, &pipeline_instructions[%u]);\n"
11867                 "\tthread_ip_reset(p, t);\n"
11868                 "\tinstr_rx_exec(p);\n"
11869                 "\treturn;\n",
11870                 instr_type_to_func(instr),
11871                 instr_pos);
11872 }
11873
11874 static int
11875 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11876                            struct instruction_group_list *igl,
11877                            uint32_t jmp_instr_id,
11878                            struct instruction *jmp_instr,
11879                            struct instruction_data *jmp_data,
11880                            FILE *f)
11881 {
11882         struct instruction_group *jmp_g, *g;
11883         struct instruction_data *data;
11884         uint32_t instr_id;
11885
11886         switch (jmp_instr->type) {
11887         case INSTR_JMP:
11888                 break;
11889
11890         case INSTR_JMP_VALID:
11891                 fprintf(f,
11892                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11893                         jmp_instr_id);
11894                 break;
11895
11896         case INSTR_JMP_INVALID:
11897                 fprintf(f,
11898                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11899                         jmp_instr_id);
11900                 break;
11901
11902         case INSTR_JMP_HIT:
11903                 fprintf(f,
11904                         "if (t->hit)\n");
11905                 break;
11906
11907         case INSTR_JMP_MISS:
11908                 fprintf(f,
11909                         "if (!t->hit)\n");
11910                 break;
11911
11912         case INSTR_JMP_ACTION_HIT:
11913                 fprintf(f,
11914                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
11915                         jmp_instr_id);
11916                 break;
11917
11918         case INSTR_JMP_ACTION_MISS:
11919                 fprintf(f,
11920                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
11921                         jmp_instr_id);
11922                 break;
11923
11924         case INSTR_JMP_EQ:
11925                 fprintf(f,
11926                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11927                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11928                         jmp_instr_id,
11929                         jmp_instr_id);
11930                 break;
11931
11932         case INSTR_JMP_EQ_MH:
11933                 fprintf(f,
11934                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11935                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11936                         jmp_instr_id,
11937                         jmp_instr_id);
11938                 break;
11939
11940         case INSTR_JMP_EQ_HM:
11941                 fprintf(f,
11942                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11943                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11944                         jmp_instr_id,
11945                         jmp_instr_id);
11946                 break;
11947
11948         case INSTR_JMP_EQ_HH:
11949                 fprintf(f,
11950                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11951                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11952                         jmp_instr_id,
11953                         jmp_instr_id);
11954                 break;
11955
11956         case INSTR_JMP_EQ_I:
11957                 fprintf(f,
11958                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11959                         "pipeline_instructions[%u].jmp.b_val)",
11960                         jmp_instr_id,
11961                         jmp_instr_id);
11962                 break;
11963
11964         case INSTR_JMP_NEQ:
11965                 fprintf(f,
11966                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11967                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11968                         jmp_instr_id,
11969                         jmp_instr_id);
11970                 break;
11971
11972         case INSTR_JMP_NEQ_MH:
11973                 fprintf(f,
11974                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11975                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11976                         jmp_instr_id,
11977                         jmp_instr_id);
11978                 break;
11979
11980         case INSTR_JMP_NEQ_HM:
11981                 fprintf(f,
11982                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11983                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11984                         jmp_instr_id,
11985                         jmp_instr_id);
11986                 break;
11987
11988         case INSTR_JMP_NEQ_HH:
11989                 fprintf(f,
11990                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11991                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11992                         jmp_instr_id,
11993                         jmp_instr_id);
11994                 break;
11995
11996         case INSTR_JMP_NEQ_I:
11997                 fprintf(f,
11998                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11999                         "pipeline_instructions[%u].jmp.b_val)",
12000                         jmp_instr_id,
12001                         jmp_instr_id);
12002                 break;
12003
12004         case INSTR_JMP_LT:
12005                 fprintf(f,
12006                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12007                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12008                         jmp_instr_id,
12009                         jmp_instr_id);
12010                 break;
12011
12012         case INSTR_JMP_LT_MH:
12013                 fprintf(f,
12014                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12015                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12016                         jmp_instr_id,
12017                         jmp_instr_id);
12018                 break;
12019
12020         case INSTR_JMP_LT_HM:
12021                 fprintf(f,
12022                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12023                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12024                         jmp_instr_id,
12025                         jmp_instr_id);
12026                 break;
12027
12028         case INSTR_JMP_LT_HH:
12029                 fprintf(f,
12030                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12031                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12032                         jmp_instr_id,
12033                         jmp_instr_id);
12034                 break;
12035
12036         case INSTR_JMP_LT_MI:
12037                 fprintf(f,
12038                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12039                         "pipeline_instructions[%u].jmp.b_val)",
12040                         jmp_instr_id,
12041                         jmp_instr_id);
12042                 break;
12043
12044         case INSTR_JMP_LT_HI:
12045                 fprintf(f,
12046                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12047                         "pipeline_instructions[%u].jmp.b_val)",
12048                         jmp_instr_id,
12049                         jmp_instr_id);
12050                 break;
12051
12052         case INSTR_JMP_GT:
12053                 fprintf(f,
12054                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12055                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12056                         jmp_instr_id,
12057                         jmp_instr_id);
12058                 break;
12059
12060         case INSTR_JMP_GT_MH:
12061                 fprintf(f,
12062                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12063                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12064                         jmp_instr_id,
12065                         jmp_instr_id);
12066                 break;
12067
12068         case INSTR_JMP_GT_HM:
12069                 fprintf(f,
12070                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12071                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12072                         jmp_instr_id,
12073                         jmp_instr_id);
12074                 break;
12075
12076         case INSTR_JMP_GT_HH:
12077                 fprintf(f,
12078                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12079                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12080                         jmp_instr_id,
12081                         jmp_instr_id);
12082                 break;
12083
12084         case INSTR_JMP_GT_MI:
12085                 fprintf(f,
12086                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12087                         "pipeline_instructions[%u].jmp.b_val)",
12088                         jmp_instr_id,
12089                         jmp_instr_id);
12090                 break;
12091
12092         case INSTR_JMP_GT_HI:
12093                 fprintf(f,
12094                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12095                         "pipeline_instructions[%u].jmp.b_val)",
12096                         jmp_instr_id,
12097                         jmp_instr_id);
12098                 break;
12099
12100         default:
12101                 break;
12102         }
12103
12104         /* Find the instruction group of the jump instruction. */
12105         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12106         if (!jmp_g)
12107                 return -EINVAL;
12108
12109         /* Find the instruction group of the jump destination instruction. */
12110         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12111         if (!data)
12112                 return -EINVAL;
12113
12114         instr_id = data - p->instruction_data;
12115
12116         g = instruction_group_list_group_find(igl, instr_id);
12117         if (!g)
12118                 return -EINVAL;
12119
12120         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12121          * instruction group).
12122          */
12123         if (g->group_id == jmp_g->group_id)
12124                 fprintf(f,
12125                         "\n\t\tgoto %s;\n",
12126                         jmp_data->jmp_label);
12127         else
12128                 fprintf(f,
12129                         " {\n"
12130                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12131                         "\t\treturn;\n"
12132                         "\t}\n\n",
12133                         g->group_id);
12134
12135         return 0;
12136 }
12137
12138 static void
12139 instruction_group_list_codegen(struct instruction_group_list *igl,
12140                                struct rte_swx_pipeline *p,
12141                                FILE *f)
12142 {
12143         struct instruction_group *g;
12144         uint32_t i;
12145         int is_required = 0;
12146
12147         /* Check if code generation is required. */
12148         TAILQ_FOREACH(g, igl, node)
12149                 if (g->first_instr_id < g->last_instr_id)
12150                         is_required = 1;
12151
12152         if (!is_required)
12153                 return;
12154
12155         /* Generate the code for the pipeline instruction array. */
12156         fprintf(f,
12157                 "static const struct instruction pipeline_instructions[] = {\n");
12158
12159         for (i = 0; i < p->n_instructions; i++) {
12160                 struct instruction *instr = &p->instructions[i];
12161                 instruction_export_t func = export_table[instr->type];
12162
12163                 func(instr, f);
12164         }
12165
12166         fprintf(f, "};\n\n");
12167
12168         /* Generate the code for the pipeline functions: one function for each instruction group
12169          * that contains more than one instruction.
12170          */
12171         TAILQ_FOREACH(g, igl, node) {
12172                 struct instruction *last_instr;
12173                 uint32_t j;
12174
12175                 /* Skip if group contains a single instruction. */
12176                 if (g->last_instr_id == g->first_instr_id)
12177                         continue;
12178
12179                 /* Generate new pipeline function. */
12180                 fprintf(f,
12181                         "void\n"
12182                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12183                         "{\n"
12184                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12185                         "\n",
12186                         g->group_id);
12187
12188                 /* Generate the code for each pipeline instruction. */
12189                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12190                         struct instruction *instr = &p->instructions[j];
12191                         struct instruction_data *data = &p->instruction_data[j];
12192
12193                         /* Label, if present. */
12194                         if (data->label[0])
12195                                 fprintf(f, "\n%s : ", data->label);
12196                         else
12197                                 fprintf(f, "\n\t");
12198
12199                         /* TX instruction type. */
12200                         if (instruction_does_tx(instr)) {
12201                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12202                                 continue;
12203                         }
12204
12205                         /* Jump instruction type. */
12206                         if (instruction_is_jmp(instr)) {
12207                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12208                                 continue;
12209                         }
12210
12211                         /* Any other instruction type. */
12212                         fprintf(f,
12213                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12214                                 instr_type_to_func(instr),
12215                                 j);
12216                 }
12217
12218                 /* Finalize the generated pipeline function. For some instructions such as TX,
12219                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12220                  * decided unconditionally and the instruction pointer of the current thread set
12221                  * accordingly; for all the other instructions, the instruction pointer must be
12222                  * incremented now.
12223                  */
12224                 last_instr = &p->instructions[g->last_instr_id];
12225
12226                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12227                         fprintf(f,
12228                                 "thread_ip_inc(p);\n");
12229
12230                 fprintf(f,
12231                         "}\n"
12232                         "\n");
12233         }
12234 }
12235
12236 static uint32_t
12237 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12238 {
12239         struct instruction_group *g;
12240         uint32_t n_custom_instr = 0;
12241
12242         /* Groups with a single instruction: no function is generated for this group, the group
12243          * keeps its current instruction. Groups with more than two instructions: one function and
12244          * the associated custom instruction get generated for each such group.
12245          */
12246         TAILQ_FOREACH(g, igl, node) {
12247                 if (g->first_instr_id == g->last_instr_id)
12248                         continue;
12249
12250                 n_custom_instr++;
12251         }
12252
12253         return n_custom_instr;
12254 }
12255
12256 static int
12257 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12258 {
12259         struct action *a;
12260         FILE *f = NULL;
12261
12262         /* Create the .c file. */
12263         f = fopen("/tmp/pipeline.c", "w");
12264         if (!f)
12265                 return -EIO;
12266
12267         /* Include the .h file. */
12268         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12269
12270         /* Add the code for each action. */
12271         TAILQ_FOREACH(a, &p->actions, node) {
12272                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12273
12274                 action_data_codegen(a, f);
12275
12276                 fprintf(f, "\n");
12277
12278                 action_instr_codegen(a, f);
12279
12280                 fprintf(f, "\n");
12281         }
12282
12283         /* Add the pipeline code. */
12284         instruction_group_list_codegen(igl, p, f);
12285
12286         /* Close the .c file. */
12287         fclose(f);
12288
12289         return 0;
12290 }
12291
12292 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12293 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12294 #endif
12295
12296 static int
12297 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12298 {
12299         struct action *a;
12300         struct instruction_group *g;
12301         char *dir_in, *buffer = NULL;
12302         const char *dir_out;
12303         int status = 0;
12304
12305         /* Get the environment variables. */
12306         dir_in = getenv("RTE_INSTALL_DIR");
12307         if (!dir_in) {
12308                 status = -EINVAL;
12309                 goto free;
12310         }
12311
12312         dir_out = "/tmp";
12313
12314         /* Memory allocation for the command buffer. */
12315         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12316         if (!buffer) {
12317                 status = -ENOMEM;
12318                 goto free;
12319         }
12320
12321         snprintf(buffer,
12322                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12323                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12324                  "-I %s/lib/pipeline "
12325                  "-I %s/lib/eal/include "
12326                  "-I %s/lib/eal/x86/include "
12327                  "-I %s/lib/eal/include/generic "
12328                  "-I %s/lib/meter "
12329                  "-I %s/lib/port "
12330                  "-I %s/lib/table "
12331                  "-I %s/lib/pipeline "
12332                  "-I %s/config "
12333                  "-I %s/build "
12334                  "-I %s/lib/eal/linux/include "
12335                  ">%s/pipeline.log 2>&1 "
12336                  "&& "
12337                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12338                  ">>%s/pipeline.log 2>&1",
12339                  dir_out,
12340                  dir_out,
12341                  dir_in,
12342                  dir_in,
12343                  dir_in,
12344                  dir_in,
12345                  dir_in,
12346                  dir_in,
12347                  dir_in,
12348                  dir_in,
12349                  dir_in,
12350                  dir_in,
12351                  dir_in,
12352                  dir_out,
12353                  dir_out,
12354                  dir_out,
12355                  dir_out);
12356
12357         /* Build the shared object library. */
12358         status = system(buffer);
12359         if (status)
12360                 goto free;
12361
12362         /* Open library. */
12363         snprintf(buffer,
12364                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12365                  "%s/libpipeline.so",
12366                  dir_out);
12367
12368         p->lib = dlopen(buffer, RTLD_LAZY);
12369         if (!p->lib) {
12370                 status = -EIO;
12371                 goto free;
12372         }
12373
12374         /* Get the action function symbols. */
12375         TAILQ_FOREACH(a, &p->actions, node) {
12376                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12377
12378                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12379                 if (!p->action_funcs[a->id]) {
12380                         status = -EINVAL;
12381                         goto free;
12382                 }
12383         }
12384
12385         /* Get the pipeline function symbols. */
12386         TAILQ_FOREACH(g, igl, node) {
12387                 if (g->first_instr_id == g->last_instr_id)
12388                         continue;
12389
12390                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12391
12392                 g->func = dlsym(p->lib, buffer);
12393                 if (!g->func) {
12394                         status = -EINVAL;
12395                         goto free;
12396                 }
12397         }
12398
12399 free:
12400         if (status && p->lib) {
12401                 dlclose(p->lib);
12402                 p->lib = NULL;
12403         }
12404
12405         free(buffer);
12406
12407         return status;
12408 }
12409
12410 static int
12411 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12412                       struct instruction_group_list *igl)
12413 {
12414         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12415
12416         /* Check that enough space is available within the pipeline instruction table to store all
12417          * the custom instructions.
12418          */
12419         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12420                 return -ENOSPC;
12421
12422         return 0;
12423 }
12424
12425 static void
12426 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12427 {
12428         struct instruction_group *g;
12429         uint32_t i;
12430
12431         /* Pipeline table instructions. */
12432         for (i = 0; i < p->n_instructions; i++) {
12433                 struct instruction *instr = &p->instructions[i];
12434
12435                 if (instr->type == INSTR_TABLE)
12436                         instr->type = INSTR_TABLE_AF;
12437
12438                 if (instr->type == INSTR_LEARNER)
12439                         instr->type = INSTR_LEARNER_AF;
12440         }
12441
12442         /* Pipeline custom instructions. */
12443         i = 0;
12444         TAILQ_FOREACH(g, igl, node) {
12445                 struct instruction *instr = &p->instructions[g->first_instr_id];
12446                 uint32_t j;
12447
12448                 if (g->first_instr_id == g->last_instr_id)
12449                         continue;
12450
12451                 /* Install a new custom instruction. */
12452                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12453
12454                 /* First instruction of the group: change its type to the new custom instruction. */
12455                 instr->type = INSTR_CUSTOM_0 + i;
12456
12457                 /* All the subsequent instructions of the group: invalidate. */
12458                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12459                         struct instruction_data *data = &p->instruction_data[j];
12460
12461                         data->invalid = 1;
12462                 }
12463
12464                 i++;
12465         }
12466
12467         /* Remove the invalidated instructions. */
12468         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12469
12470         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12471          * instructions that are the only instruction within their group, so they were left
12472          * unmodified).
12473          */
12474         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12475 }
12476
12477 static int
12478 pipeline_compile(struct rte_swx_pipeline *p)
12479 {
12480         struct instruction_group_list *igl = NULL;
12481         int status = 0;
12482
12483         igl = instruction_group_list_create(p);
12484         if (!igl) {
12485                 status = -ENOMEM;
12486                 goto free;
12487         }
12488
12489         /* Code generation. */
12490         status = pipeline_codegen(p, igl);
12491         if (status)
12492                 goto free;
12493
12494         /* Build and load the shared object library. */
12495         status = pipeline_libload(p, igl);
12496         if (status)
12497                 goto free;
12498
12499         /* Adjust instructions. */
12500         status = pipeline_adjust_check(p, igl);
12501         if (status)
12502                 goto free;
12503
12504         pipeline_adjust(p, igl);
12505
12506 free:
12507         instruction_group_list_free(igl);
12508
12509         return status;
12510 }