lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <dlfcn.h>
   8
   9 #include <rte_swx_port_ethdev.h>
  10 #include <rte_swx_port_fd.h>
  11 #include <rte_swx_port_ring.h>
  12 #include "rte_swx_port_source_sink.h"
  13
  14 #include <rte_swx_table_em.h>
  15 #include <rte_swx_table_wm.h>
  16
  17 #include "rte_swx_pipeline_internal.h"
  18
  19 #define CHECK(condition, err_code)                                             \
  20 do {                                                                           \
  21         if (!(condition))                                                      \
  22                 return -(err_code);                                            \
  23 } while (0)
  24
  25 #define CHECK_NAME(name, err_code)                                             \
  26         CHECK((name) &&                                                        \
  27               (name)[0] &&                                                     \
  28               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  29               err_code)
  30
  31 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  32         CHECK((instr) &&                                                       \
  33               (instr)[0] &&                                                    \
  34               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  35                RTE_SWX_INSTRUCTION_SIZE),                                      \
  36               err_code)
  37
  38 /*
  39  * Environment.
  40  */
  41 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  42
  43 #include <rte_malloc.h>
  44
  45 static void *
  46 env_malloc(size_t size, size_t alignment, int numa_node)
  47 {
  48         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  49 }
  50
  51 static void
  52 env_free(void *start, size_t size __rte_unused)
  53 {
  54         rte_free(start);
  55 }
  56
  57 #else
  58
  59 #include <numa.h>
  60
  61 static void *
  62 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  63 {
  64         void *start;
  65
  66         if (numa_available() == -1)
  67                 return NULL;
  68
  69         start = numa_alloc_onnode(size, numa_node);
  70         if (!start)
  71                 return NULL;
  72
  73         memset(start, 0, size);
  74         return start;
  75 }
  76
  77 static void
  78 env_free(void *start, size_t size)
  79 {
  80         if (numa_available() == -1)
  81                 return;
  82
  83         numa_free(start, size);
  84 }
  85
  86 #endif
  87
  88 /*
  89  * Struct.
  90  */
  91 static struct struct_type *
  92 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  93 {
  94         struct struct_type *elem;
  95
  96         TAILQ_FOREACH(elem, &p->struct_types, node)
  97                 if (strcmp(elem->name, name) == 0)
  98                         return elem;
  99
 100         return NULL;
 101 }
 102
 103 static struct field *
 104 struct_type_field_find(struct struct_type *st, const char *name)
 105 {
 106         uint32_t i;
 107
 108         for (i = 0; i < st->n_fields; i++) {
 109                 struct field *f = &st->fields[i];
 110
 111                 if (strcmp(f->name, name) == 0)
 112                         return f;
 113         }
 114
 115         return NULL;
 116 }
 117
 118 int
 119 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 120                                       const char *name,
 121                                       struct rte_swx_field_params *fields,
 122                                       uint32_t n_fields,
 123                                       int last_field_has_variable_size)
 124 {
 125         struct struct_type *st;
 126         uint32_t i;
 127
 128         CHECK(p, EINVAL);
 129         CHECK_NAME(name, EINVAL);
 130         CHECK(fields, EINVAL);
 131         CHECK(n_fields, EINVAL);
 132
 133         for (i = 0; i < n_fields; i++) {
 134                 struct rte_swx_field_params *f = &fields[i];
 135                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 136                 uint32_t j;
 137
 138                 CHECK_NAME(f->name, EINVAL);
 139                 CHECK(f->n_bits, EINVAL);
 140                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 141                 CHECK((f->n_bits & 7) == 0, EINVAL);
 142
 143                 for (j = 0; j < i; j++) {
 144                         struct rte_swx_field_params *f_prev = &fields[j];
 145
 146                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 147                 }
 148         }
 149
 150         CHECK(!struct_type_find(p, name), EEXIST);
 151
 152         /* Node allocation. */
 153         st = calloc(1, sizeof(struct struct_type));
 154         CHECK(st, ENOMEM);
 155
 156         st->fields = calloc(n_fields, sizeof(struct field));
 157         if (!st->fields) {
 158                 free(st);
 159                 CHECK(0, ENOMEM);
 160         }
 161
 162         /* Node initialization. */
 163         strcpy(st->name, name);
 164         for (i = 0; i < n_fields; i++) {
 165                 struct field *dst = &st->fields[i];
 166                 struct rte_swx_field_params *src = &fields[i];
 167                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 168
 169                 strcpy(dst->name, src->name);
 170                 dst->n_bits = src->n_bits;
 171                 dst->offset = st->n_bits;
 172                 dst->var_size = var_size;
 173
 174                 st->n_bits += src->n_bits;
 175                 st->n_bits_min += var_size ? 0 : src->n_bits;
 176         }
 177         st->n_fields = n_fields;
 178         st->var_size = last_field_has_variable_size;
 179
 180         /* Node add to tailq. */
 181         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 182
 183         return 0;
 184 }
 185
 186 static int
 187 struct_build(struct rte_swx_pipeline *p)
 188 {
 189         uint32_t i;
 190
 191         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 192                 struct thread *t = &p->threads[i];
 193
 194                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 195                 CHECK(t->structs, ENOMEM);
 196         }
 197
 198         return 0;
 199 }
 200
 201 static void
 202 struct_build_free(struct rte_swx_pipeline *p)
 203 {
 204         uint32_t i;
 205
 206         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 207                 struct thread *t = &p->threads[i];
 208
 209                 free(t->structs);
 210                 t->structs = NULL;
 211         }
 212 }
 213
 214 static void
 215 struct_free(struct rte_swx_pipeline *p)
 216 {
 217         struct_build_free(p);
 218
 219         /* Struct types. */
 220         for ( ; ; ) {
 221                 struct struct_type *elem;
 222
 223                 elem = TAILQ_FIRST(&p->struct_types);
 224                 if (!elem)
 225                         break;
 226
 227                 TAILQ_REMOVE(&p->struct_types, elem, node);
 228                 free(elem->fields);
 229                 free(elem);
 230         }
 231 }
 232
 233 /*
 234  * Input port.
 235  */
 236 static struct port_in_type *
 237 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 238 {
 239         struct port_in_type *elem;
 240
 241         if (!name)
 242                 return NULL;
 243
 244         TAILQ_FOREACH(elem, &p->port_in_types, node)
 245                 if (strcmp(elem->name, name) == 0)
 246                         return elem;
 247
 248         return NULL;
 249 }
 250
 251 int
 252 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 253                                        const char *name,
 254                                        struct rte_swx_port_in_ops *ops)
 255 {
 256         struct port_in_type *elem;
 257
 258         CHECK(p, EINVAL);
 259         CHECK_NAME(name, EINVAL);
 260         CHECK(ops, EINVAL);
 261         CHECK(ops->create, EINVAL);
 262         CHECK(ops->free, EINVAL);
 263         CHECK(ops->pkt_rx, EINVAL);
 264         CHECK(ops->stats_read, EINVAL);
 265
 266         CHECK(!port_in_type_find(p, name), EEXIST);
 267
 268         /* Node allocation. */
 269         elem = calloc(1, sizeof(struct port_in_type));
 270         CHECK(elem, ENOMEM);
 271
 272         /* Node initialization. */
 273         strcpy(elem->name, name);
 274         memcpy(&elem->ops, ops, sizeof(*ops));
 275
 276         /* Node add to tailq. */
 277         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 278
 279         return 0;
 280 }
 281
 282 static struct port_in *
 283 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 284 {
 285         struct port_in *port;
 286
 287         TAILQ_FOREACH(port, &p->ports_in, node)
 288                 if (port->id == port_id)
 289                         return port;
 290
 291         return NULL;
 292 }
 293
 294 int
 295 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 296                                 uint32_t port_id,
 297                                 const char *port_type_name,
 298                                 void *args)
 299 {
 300         struct port_in_type *type = NULL;
 301         struct port_in *port = NULL;
 302         void *obj = NULL;
 303
 304         CHECK(p, EINVAL);
 305
 306         CHECK(!port_in_find(p, port_id), EINVAL);
 307
 308         CHECK_NAME(port_type_name, EINVAL);
 309         type = port_in_type_find(p, port_type_name);
 310         CHECK(type, EINVAL);
 311
 312         obj = type->ops.create(args);
 313         CHECK(obj, ENODEV);
 314
 315         /* Node allocation. */
 316         port = calloc(1, sizeof(struct port_in));
 317         CHECK(port, ENOMEM);
 318
 319         /* Node initialization. */
 320         port->type = type;
 321         port->obj = obj;
 322         port->id = port_id;
 323
 324         /* Node add to tailq. */
 325         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 326         if (p->n_ports_in < port_id + 1)
 327                 p->n_ports_in = port_id + 1;
 328
 329         return 0;
 330 }
 331
 332 static int
 333 port_in_build(struct rte_swx_pipeline *p)
 334 {
 335         struct port_in *port;
 336         uint32_t i;
 337
 338         CHECK(p->n_ports_in, EINVAL);
 339         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 340
 341         for (i = 0; i < p->n_ports_in; i++)
 342                 CHECK(port_in_find(p, i), EINVAL);
 343
 344         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 345         CHECK(p->in, ENOMEM);
 346
 347         TAILQ_FOREACH(port, &p->ports_in, node) {
 348                 struct port_in_runtime *in = &p->in[port->id];
 349
 350                 in->pkt_rx = port->type->ops.pkt_rx;
 351                 in->obj = port->obj;
 352         }
 353
 354         return 0;
 355 }
 356
 357 static void
 358 port_in_build_free(struct rte_swx_pipeline *p)
 359 {
 360         free(p->in);
 361         p->in = NULL;
 362 }
 363
 364 static void
 365 port_in_free(struct rte_swx_pipeline *p)
 366 {
 367         port_in_build_free(p);
 368
 369         /* Input ports. */
 370         for ( ; ; ) {
 371                 struct port_in *port;
 372
 373                 port = TAILQ_FIRST(&p->ports_in);
 374                 if (!port)
 375                         break;
 376
 377                 TAILQ_REMOVE(&p->ports_in, port, node);
 378                 port->type->ops.free(port->obj);
 379                 free(port);
 380         }
 381
 382         /* Input port types. */
 383         for ( ; ; ) {
 384                 struct port_in_type *elem;
 385
 386                 elem = TAILQ_FIRST(&p->port_in_types);
 387                 if (!elem)
 388                         break;
 389
 390                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 391                 free(elem);
 392         }
 393 }
 394
 395 /*
 396  * Output port.
 397  */
 398 static struct port_out_type *
 399 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 400 {
 401         struct port_out_type *elem;
 402
 403         if (!name)
 404                 return NULL;
 405
 406         TAILQ_FOREACH(elem, &p->port_out_types, node)
 407                 if (!strcmp(elem->name, name))
 408                         return elem;
 409
 410         return NULL;
 411 }
 412
 413 int
 414 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 415                                         const char *name,
 416                                         struct rte_swx_port_out_ops *ops)
 417 {
 418         struct port_out_type *elem;
 419
 420         CHECK(p, EINVAL);
 421         CHECK_NAME(name, EINVAL);
 422         CHECK(ops, EINVAL);
 423         CHECK(ops->create, EINVAL);
 424         CHECK(ops->free, EINVAL);
 425         CHECK(ops->pkt_tx, EINVAL);
 426         CHECK(ops->pkt_fast_clone_tx, EINVAL);
 427         CHECK(ops->pkt_clone_tx, EINVAL);
 428         CHECK(ops->stats_read, EINVAL);
 429
 430         CHECK(!port_out_type_find(p, name), EEXIST);
 431
 432         /* Node allocation. */
 433         elem = calloc(1, sizeof(struct port_out_type));
 434         CHECK(elem, ENOMEM);
 435
 436         /* Node initialization. */
 437         strcpy(elem->name, name);
 438         memcpy(&elem->ops, ops, sizeof(*ops));
 439
 440         /* Node add to tailq. */
 441         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 442
 443         return 0;
 444 }
 445
 446 static struct port_out *
 447 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 448 {
 449         struct port_out *port;
 450
 451         TAILQ_FOREACH(port, &p->ports_out, node)
 452                 if (port->id == port_id)
 453                         return port;
 454
 455         return NULL;
 456 }
 457
 458 int
 459 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 460                                  uint32_t port_id,
 461                                  const char *port_type_name,
 462                                  void *args)
 463 {
 464         struct port_out_type *type = NULL;
 465         struct port_out *port = NULL;
 466         void *obj = NULL;
 467
 468         CHECK(p, EINVAL);
 469
 470         CHECK(!port_out_find(p, port_id), EINVAL);
 471
 472         CHECK_NAME(port_type_name, EINVAL);
 473         type = port_out_type_find(p, port_type_name);
 474         CHECK(type, EINVAL);
 475
 476         obj = type->ops.create(args);
 477         CHECK(obj, ENODEV);
 478
 479         /* Node allocation. */
 480         port = calloc(1, sizeof(struct port_out));
 481         CHECK(port, ENOMEM);
 482
 483         /* Node initialization. */
 484         port->type = type;
 485         port->obj = obj;
 486         port->id = port_id;
 487
 488         /* Node add to tailq. */
 489         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 490         if (p->n_ports_out < port_id + 1)
 491                 p->n_ports_out = port_id + 1;
 492
 493         return 0;
 494 }
 495
 496 static int
 497 port_out_build(struct rte_swx_pipeline *p)
 498 {
 499         struct port_out *port;
 500         uint32_t i;
 501
 502         CHECK(p->n_ports_out, EINVAL);
 503
 504         for (i = 0; i < p->n_ports_out; i++)
 505                 CHECK(port_out_find(p, i), EINVAL);
 506
 507         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 508         CHECK(p->out, ENOMEM);
 509
 510         TAILQ_FOREACH(port, &p->ports_out, node) {
 511                 struct port_out_runtime *out = &p->out[port->id];
 512
 513                 out->pkt_tx = port->type->ops.pkt_tx;
 514                 out->pkt_fast_clone_tx = port->type->ops.pkt_fast_clone_tx;
 515                 out->pkt_clone_tx = port->type->ops.pkt_clone_tx;
 516                 out->flush = port->type->ops.flush;
 517                 out->obj = port->obj;
 518         }
 519
 520         return 0;
 521 }
 522
 523 static void
 524 port_out_build_free(struct rte_swx_pipeline *p)
 525 {
 526         free(p->out);
 527         p->out = NULL;
 528 }
 529
 530 static void
 531 port_out_free(struct rte_swx_pipeline *p)
 532 {
 533         port_out_build_free(p);
 534
 535         /* Output ports. */
 536         for ( ; ; ) {
 537                 struct port_out *port;
 538
 539                 port = TAILQ_FIRST(&p->ports_out);
 540                 if (!port)
 541                         break;
 542
 543                 TAILQ_REMOVE(&p->ports_out, port, node);
 544                 port->type->ops.free(port->obj);
 545                 free(port);
 546         }
 547
 548         /* Output port types. */
 549         for ( ; ; ) {
 550                 struct port_out_type *elem;
 551
 552                 elem = TAILQ_FIRST(&p->port_out_types);
 553                 if (!elem)
 554                         break;
 555
 556                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 557                 free(elem);
 558         }
 559 }
 560
 561 /*
 562  * Packet mirroring.
 563  */
 564 int
 565 rte_swx_pipeline_mirroring_config(struct rte_swx_pipeline *p,
 566                                   struct rte_swx_pipeline_mirroring_params *params)
 567 {
 568         CHECK(p, EINVAL);
 569         CHECK(params, EINVAL);
 570         CHECK(params->n_slots, EINVAL);
 571         CHECK(params->n_sessions, EINVAL);
 572         CHECK(!p->build_done, EEXIST);
 573
 574         p->n_mirroring_slots = rte_align32pow2(params->n_slots);
 575         if (p->n_mirroring_slots > 64)
 576                 p->n_mirroring_slots = 64;
 577
 578         p->n_mirroring_sessions = rte_align32pow2(params->n_sessions);
 579
 580         return 0;
 581 }
 582
 583 static void
 584 mirroring_build_free(struct rte_swx_pipeline *p)
 585 {
 586         uint32_t i;
 587
 588         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 589                 struct thread *t = &p->threads[i];
 590
 591                 /* mirroring_slots. */
 592                 free(t->mirroring_slots);
 593                 t->mirroring_slots = NULL;
 594         }
 595
 596         /* mirroring_sessions. */
 597         free(p->mirroring_sessions);
 598         p->mirroring_sessions = NULL;
 599 }
 600
 601 static void
 602 mirroring_free(struct rte_swx_pipeline *p)
 603 {
 604         mirroring_build_free(p);
 605 }
 606
 607 static int
 608 mirroring_build(struct rte_swx_pipeline *p)
 609 {
 610         uint32_t i;
 611
 612         if (!p->n_mirroring_slots || !p->n_mirroring_sessions)
 613                 return 0;
 614
 615         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 616                 struct thread *t = &p->threads[i];
 617
 618                 /* mirroring_slots. */
 619                 t->mirroring_slots = calloc(p->n_mirroring_slots, sizeof(uint32_t));
 620                 if (!t->mirroring_slots)
 621                         goto error;
 622         }
 623
 624         /* mirroring_sessions. */
 625         p->mirroring_sessions = calloc(p->n_mirroring_sessions, sizeof(struct mirroring_session));
 626         if (!p->mirroring_sessions)
 627                 goto error;
 628
 629         return 0;
 630
 631 error:
 632         mirroring_build_free(p);
 633         return -ENOMEM;
 634 }
 635
 636 /*
 637  * Extern object.
 638  */
 639 static struct extern_type *
 640 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 641 {
 642         struct extern_type *elem;
 643
 644         TAILQ_FOREACH(elem, &p->extern_types, node)
 645                 if (strcmp(elem->name, name) == 0)
 646                         return elem;
 647
 648         return NULL;
 649 }
 650
 651 static struct extern_type_member_func *
 652 extern_type_member_func_find(struct extern_type *type, const char *name)
 653 {
 654         struct extern_type_member_func *elem;
 655
 656         TAILQ_FOREACH(elem, &type->funcs, node)
 657                 if (strcmp(elem->name, name) == 0)
 658                         return elem;
 659
 660         return NULL;
 661 }
 662
 663 static struct extern_obj *
 664 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 665 {
 666         struct extern_obj *elem;
 667
 668         TAILQ_FOREACH(elem, &p->extern_objs, node)
 669                 if (strcmp(elem->name, name) == 0)
 670                         return elem;
 671
 672         return NULL;
 673 }
 674
 675 static struct extern_type_member_func *
 676 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 677                              const char *name,
 678                              struct extern_obj **obj)
 679 {
 680         struct extern_obj *object;
 681         struct extern_type_member_func *func;
 682         char *object_name, *func_name;
 683
 684         if (name[0] != 'e' || name[1] != '.')
 685                 return NULL;
 686
 687         object_name = strdup(&name[2]);
 688         if (!object_name)
 689                 return NULL;
 690
 691         func_name = strchr(object_name, '.');
 692         if (!func_name) {
 693                 free(object_name);
 694                 return NULL;
 695         }
 696
 697         *func_name = 0;
 698         func_name++;
 699
 700         object = extern_obj_find(p, object_name);
 701         if (!object) {
 702                 free(object_name);
 703                 return NULL;
 704         }
 705
 706         func = extern_type_member_func_find(object->type, func_name);
 707         if (!func) {
 708                 free(object_name);
 709                 return NULL;
 710         }
 711
 712         if (obj)
 713                 *obj = object;
 714
 715         free(object_name);
 716         return func;
 717 }
 718
 719 static struct field *
 720 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 721                                const char *name,
 722                                struct extern_obj **object)
 723 {
 724         struct extern_obj *obj;
 725         struct field *f;
 726         char *obj_name, *field_name;
 727
 728         if ((name[0] != 'e') || (name[1] != '.'))
 729                 return NULL;
 730
 731         obj_name = strdup(&name[2]);
 732         if (!obj_name)
 733                 return NULL;
 734
 735         field_name = strchr(obj_name, '.');
 736         if (!field_name) {
 737                 free(obj_name);
 738                 return NULL;
 739         }
 740
 741         *field_name = 0;
 742         field_name++;
 743
 744         obj = extern_obj_find(p, obj_name);
 745         if (!obj) {
 746                 free(obj_name);
 747                 return NULL;
 748         }
 749
 750         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 751         if (!f) {
 752                 free(obj_name);
 753                 return NULL;
 754         }
 755
 756         if (object)
 757                 *object = obj;
 758
 759         free(obj_name);
 760         return f;
 761 }
 762
 763 int
 764 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 765         const char *name,
 766         const char *mailbox_struct_type_name,
 767         rte_swx_extern_type_constructor_t constructor,
 768         rte_swx_extern_type_destructor_t destructor)
 769 {
 770         struct extern_type *elem;
 771         struct struct_type *mailbox_struct_type;
 772
 773         CHECK(p, EINVAL);
 774
 775         CHECK_NAME(name, EINVAL);
 776         CHECK(!extern_type_find(p, name), EEXIST);
 777
 778         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 779         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 780         CHECK(mailbox_struct_type, EINVAL);
 781         CHECK(!mailbox_struct_type->var_size, EINVAL);
 782
 783         CHECK(constructor, EINVAL);
 784         CHECK(destructor, EINVAL);
 785
 786         /* Node allocation. */
 787         elem = calloc(1, sizeof(struct extern_type));
 788         CHECK(elem, ENOMEM);
 789
 790         /* Node initialization. */
 791         strcpy(elem->name, name);
 792         elem->mailbox_struct_type = mailbox_struct_type;
 793         elem->constructor = constructor;
 794         elem->destructor = destructor;
 795         TAILQ_INIT(&elem->funcs);
 796
 797         /* Node add to tailq. */
 798         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 799
 800         return 0;
 801 }
 802
 803 int
 804 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 805         const char *extern_type_name,
 806         const char *name,
 807         rte_swx_extern_type_member_func_t member_func)
 808 {
 809         struct extern_type *type;
 810         struct extern_type_member_func *type_member;
 811
 812         CHECK(p, EINVAL);
 813
 814         CHECK_NAME(extern_type_name, EINVAL);
 815         type = extern_type_find(p, extern_type_name);
 816         CHECK(type, EINVAL);
 817         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 818
 819         CHECK_NAME(name, EINVAL);
 820         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 821
 822         CHECK(member_func, EINVAL);
 823
 824         /* Node allocation. */
 825         type_member = calloc(1, sizeof(struct extern_type_member_func));
 826         CHECK(type_member, ENOMEM);
 827
 828         /* Node initialization. */
 829         strcpy(type_member->name, name);
 830         type_member->func = member_func;
 831         type_member->id = type->n_funcs;
 832
 833         /* Node add to tailq. */
 834         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 835         type->n_funcs++;
 836
 837         return 0;
 838 }
 839
 840 int
 841 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 842                                       const char *extern_type_name,
 843                                       const char *name,
 844                                       const char *args)
 845 {
 846         struct extern_type *type;
 847         struct extern_obj *obj;
 848         void *obj_handle;
 849
 850         CHECK(p, EINVAL);
 851
 852         CHECK_NAME(extern_type_name, EINVAL);
 853         type = extern_type_find(p, extern_type_name);
 854         CHECK(type, EINVAL);
 855
 856         CHECK_NAME(name, EINVAL);
 857         CHECK(!extern_obj_find(p, name), EEXIST);
 858
 859         /* Node allocation. */
 860         obj = calloc(1, sizeof(struct extern_obj));
 861         CHECK(obj, ENOMEM);
 862
 863         /* Object construction. */
 864         obj_handle = type->constructor(args);
 865         if (!obj_handle) {
 866                 free(obj);
 867                 CHECK(0, ENODEV);
 868         }
 869
 870         /* Node initialization. */
 871         strcpy(obj->name, name);
 872         obj->type = type;
 873         obj->obj = obj_handle;
 874         obj->struct_id = p->n_structs;
 875         obj->id = p->n_extern_objs;
 876
 877         /* Node add to tailq. */
 878         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 879         p->n_extern_objs++;
 880         p->n_structs++;
 881
 882         return 0;
 883 }
 884
 885 static int
 886 extern_obj_build(struct rte_swx_pipeline *p)
 887 {
 888         uint32_t i;
 889
 890         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 891                 struct thread *t = &p->threads[i];
 892                 struct extern_obj *obj;
 893
 894                 t->extern_objs = calloc(p->n_extern_objs,
 895                                         sizeof(struct extern_obj_runtime));
 896                 CHECK(t->extern_objs, ENOMEM);
 897
 898                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 899                         struct extern_obj_runtime *r =
 900                                 &t->extern_objs[obj->id];
 901                         struct extern_type_member_func *func;
 902                         uint32_t mailbox_size =
 903                                 obj->type->mailbox_struct_type->n_bits / 8;
 904
 905                         r->obj = obj->obj;
 906
 907                         r->mailbox = calloc(1, mailbox_size);
 908                         CHECK(r->mailbox, ENOMEM);
 909
 910                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 911                                 r->funcs[func->id] = func->func;
 912
 913                         t->structs[obj->struct_id] = r->mailbox;
 914                 }
 915         }
 916
 917         return 0;
 918 }
 919
 920 static void
 921 extern_obj_build_free(struct rte_swx_pipeline *p)
 922 {
 923         uint32_t i;
 924
 925         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 926                 struct thread *t = &p->threads[i];
 927                 uint32_t j;
 928
 929                 if (!t->extern_objs)
 930                         continue;
 931
 932                 for (j = 0; j < p->n_extern_objs; j++) {
 933                         struct extern_obj_runtime *r = &t->extern_objs[j];
 934
 935                         free(r->mailbox);
 936                 }
 937
 938                 free(t->extern_objs);
 939                 t->extern_objs = NULL;
 940         }
 941 }
 942
 943 static void
 944 extern_obj_free(struct rte_swx_pipeline *p)
 945 {
 946         extern_obj_build_free(p);
 947
 948         /* Extern objects. */
 949         for ( ; ; ) {
 950                 struct extern_obj *elem;
 951
 952                 elem = TAILQ_FIRST(&p->extern_objs);
 953                 if (!elem)
 954                         break;
 955
 956                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 957                 if (elem->obj)
 958                         elem->type->destructor(elem->obj);
 959                 free(elem);
 960         }
 961
 962         /* Extern types. */
 963         for ( ; ; ) {
 964                 struct extern_type *elem;
 965
 966                 elem = TAILQ_FIRST(&p->extern_types);
 967                 if (!elem)
 968                         break;
 969
 970                 TAILQ_REMOVE(&p->extern_types, elem, node);
 971
 972                 for ( ; ; ) {
 973                         struct extern_type_member_func *func;
 974
 975                         func = TAILQ_FIRST(&elem->funcs);
 976                         if (!func)
 977                                 break;
 978
 979                         TAILQ_REMOVE(&elem->funcs, func, node);
 980                         free(func);
 981                 }
 982
 983                 free(elem);
 984         }
 985 }
 986
 987 /*
 988  * Extern function.
 989  */
 990 static struct extern_func *
 991 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 992 {
 993         struct extern_func *elem;
 994
 995         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 996                 if (strcmp(elem->name, name) == 0)
 997                         return elem;
 998
 999         return NULL;
1000 }
1001
1002 static struct extern_func *
1003 extern_func_parse(struct rte_swx_pipeline *p,
1004                   const char *name)
1005 {
1006         if (name[0] != 'f' || name[1] != '.')
1007                 return NULL;
1008
1009         return extern_func_find(p, &name[2]);
1010 }
1011
1012 static struct field *
1013 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1014                                 const char *name,
1015                                 struct extern_func **function)
1016 {
1017         struct extern_func *func;
1018         struct field *f;
1019         char *func_name, *field_name;
1020
1021         if ((name[0] != 'f') || (name[1] != '.'))
1022                 return NULL;
1023
1024         func_name = strdup(&name[2]);
1025         if (!func_name)
1026                 return NULL;
1027
1028         field_name = strchr(func_name, '.');
1029         if (!field_name) {
1030                 free(func_name);
1031                 return NULL;
1032         }
1033
1034         *field_name = 0;
1035         field_name++;
1036
1037         func = extern_func_find(p, func_name);
1038         if (!func) {
1039                 free(func_name);
1040                 return NULL;
1041         }
1042
1043         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1044         if (!f) {
1045                 free(func_name);
1046                 return NULL;
1047         }
1048
1049         if (function)
1050                 *function = func;
1051
1052         free(func_name);
1053         return f;
1054 }
1055
1056 int
1057 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1058                                       const char *name,
1059                                       const char *mailbox_struct_type_name,
1060                                       rte_swx_extern_func_t func)
1061 {
1062         struct extern_func *f;
1063         struct struct_type *mailbox_struct_type;
1064
1065         CHECK(p, EINVAL);
1066
1067         CHECK_NAME(name, EINVAL);
1068         CHECK(!extern_func_find(p, name), EEXIST);
1069
1070         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1071         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1072         CHECK(mailbox_struct_type, EINVAL);
1073         CHECK(!mailbox_struct_type->var_size, EINVAL);
1074
1075         CHECK(func, EINVAL);
1076
1077         /* Node allocation. */
1078         f = calloc(1, sizeof(struct extern_func));
1079         CHECK(func, ENOMEM);
1080
1081         /* Node initialization. */
1082         strcpy(f->name, name);
1083         f->mailbox_struct_type = mailbox_struct_type;
1084         f->func = func;
1085         f->struct_id = p->n_structs;
1086         f->id = p->n_extern_funcs;
1087
1088         /* Node add to tailq. */
1089         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1090         p->n_extern_funcs++;
1091         p->n_structs++;
1092
1093         return 0;
1094 }
1095
1096 static int
1097 extern_func_build(struct rte_swx_pipeline *p)
1098 {
1099         uint32_t i;
1100
1101         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1102                 struct thread *t = &p->threads[i];
1103                 struct extern_func *func;
1104
1105                 /* Memory allocation. */
1106                 t->extern_funcs = calloc(p->n_extern_funcs,
1107                                          sizeof(struct extern_func_runtime));
1108                 CHECK(t->extern_funcs, ENOMEM);
1109
1110                 /* Extern function. */
1111                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1112                         struct extern_func_runtime *r =
1113                                 &t->extern_funcs[func->id];
1114                         uint32_t mailbox_size =
1115                                 func->mailbox_struct_type->n_bits / 8;
1116
1117                         r->func = func->func;
1118
1119                         r->mailbox = calloc(1, mailbox_size);
1120                         CHECK(r->mailbox, ENOMEM);
1121
1122                         t->structs[func->struct_id] = r->mailbox;
1123                 }
1124         }
1125
1126         return 0;
1127 }
1128
1129 static void
1130 extern_func_build_free(struct rte_swx_pipeline *p)
1131 {
1132         uint32_t i;
1133
1134         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1135                 struct thread *t = &p->threads[i];
1136                 uint32_t j;
1137
1138                 if (!t->extern_funcs)
1139                         continue;
1140
1141                 for (j = 0; j < p->n_extern_funcs; j++) {
1142                         struct extern_func_runtime *r = &t->extern_funcs[j];
1143
1144                         free(r->mailbox);
1145                 }
1146
1147                 free(t->extern_funcs);
1148                 t->extern_funcs = NULL;
1149         }
1150 }
1151
1152 static void
1153 extern_func_free(struct rte_swx_pipeline *p)
1154 {
1155         extern_func_build_free(p);
1156
1157         for ( ; ; ) {
1158                 struct extern_func *elem;
1159
1160                 elem = TAILQ_FIRST(&p->extern_funcs);
1161                 if (!elem)
1162                         break;
1163
1164                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1165                 free(elem);
1166         }
1167 }
1168
1169 /*
1170  * Header.
1171  */
1172 static struct header *
1173 header_find(struct rte_swx_pipeline *p, const char *name)
1174 {
1175         struct header *elem;
1176
1177         TAILQ_FOREACH(elem, &p->headers, node)
1178                 if (strcmp(elem->name, name) == 0)
1179                         return elem;
1180
1181         return NULL;
1182 }
1183
1184 static struct header *
1185 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1186 {
1187         struct header *elem;
1188
1189         TAILQ_FOREACH(elem, &p->headers, node)
1190                 if (elem->struct_id == struct_id)
1191                         return elem;
1192
1193         return NULL;
1194 }
1195
1196 static struct header *
1197 header_parse(struct rte_swx_pipeline *p,
1198              const char *name)
1199 {
1200         if (name[0] != 'h' || name[1] != '.')
1201                 return NULL;
1202
1203         return header_find(p, &name[2]);
1204 }
1205
1206 static struct field *
1207 header_field_parse(struct rte_swx_pipeline *p,
1208                    const char *name,
1209                    struct header **header)
1210 {
1211         struct header *h;
1212         struct field *f;
1213         char *header_name, *field_name;
1214
1215         if ((name[0] != 'h') || (name[1] != '.'))
1216                 return NULL;
1217
1218         header_name = strdup(&name[2]);
1219         if (!header_name)
1220                 return NULL;
1221
1222         field_name = strchr(header_name, '.');
1223         if (!field_name) {
1224                 free(header_name);
1225                 return NULL;
1226         }
1227
1228         *field_name = 0;
1229         field_name++;
1230
1231         h = header_find(p, header_name);
1232         if (!h) {
1233                 free(header_name);
1234                 return NULL;
1235         }
1236
1237         f = struct_type_field_find(h->st, field_name);
1238         if (!f) {
1239                 free(header_name);
1240                 return NULL;
1241         }
1242
1243         if (header)
1244                 *header = h;
1245
1246         free(header_name);
1247         return f;
1248 }
1249
1250 int
1251 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1252                                         const char *name,
1253                                         const char *struct_type_name)
1254 {
1255         struct struct_type *st;
1256         struct header *h;
1257         size_t n_headers_max;
1258
1259         CHECK(p, EINVAL);
1260         CHECK_NAME(name, EINVAL);
1261         CHECK_NAME(struct_type_name, EINVAL);
1262
1263         CHECK(!header_find(p, name), EEXIST);
1264
1265         st = struct_type_find(p, struct_type_name);
1266         CHECK(st, EINVAL);
1267
1268         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1269         CHECK(p->n_headers < n_headers_max, ENOSPC);
1270
1271         /* Node allocation. */
1272         h = calloc(1, sizeof(struct header));
1273         CHECK(h, ENOMEM);
1274
1275         /* Node initialization. */
1276         strcpy(h->name, name);
1277         h->st = st;
1278         h->struct_id = p->n_structs;
1279         h->id = p->n_headers;
1280
1281         /* Node add to tailq. */
1282         TAILQ_INSERT_TAIL(&p->headers, h, node);
1283         p->n_headers++;
1284         p->n_structs++;
1285
1286         return 0;
1287 }
1288
1289 static int
1290 header_build(struct rte_swx_pipeline *p)
1291 {
1292         struct header *h;
1293         uint32_t n_bytes = 0, i;
1294
1295         TAILQ_FOREACH(h, &p->headers, node) {
1296                 n_bytes += h->st->n_bits / 8;
1297         }
1298
1299         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1300                 struct thread *t = &p->threads[i];
1301                 uint32_t offset = 0;
1302
1303                 t->headers = calloc(p->n_headers,
1304                                     sizeof(struct header_runtime));
1305                 CHECK(t->headers, ENOMEM);
1306
1307                 t->headers_out = calloc(p->n_headers,
1308                                         sizeof(struct header_out_runtime));
1309                 CHECK(t->headers_out, ENOMEM);
1310
1311                 t->header_storage = calloc(1, n_bytes);
1312                 CHECK(t->header_storage, ENOMEM);
1313
1314                 t->header_out_storage = calloc(1, n_bytes);
1315                 CHECK(t->header_out_storage, ENOMEM);
1316
1317                 TAILQ_FOREACH(h, &p->headers, node) {
1318                         uint8_t *header_storage;
1319                         uint32_t n_bytes =  h->st->n_bits / 8;
1320
1321                         header_storage = &t->header_storage[offset];
1322                         offset += n_bytes;
1323
1324                         t->headers[h->id].ptr0 = header_storage;
1325                         t->headers[h->id].n_bytes = n_bytes;
1326
1327                         t->structs[h->struct_id] = header_storage;
1328                 }
1329         }
1330
1331         return 0;
1332 }
1333
1334 static void
1335 header_build_free(struct rte_swx_pipeline *p)
1336 {
1337         uint32_t i;
1338
1339         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1340                 struct thread *t = &p->threads[i];
1341
1342                 free(t->headers_out);
1343                 t->headers_out = NULL;
1344
1345                 free(t->headers);
1346                 t->headers = NULL;
1347
1348                 free(t->header_out_storage);
1349                 t->header_out_storage = NULL;
1350
1351                 free(t->header_storage);
1352                 t->header_storage = NULL;
1353         }
1354 }
1355
1356 static void
1357 header_free(struct rte_swx_pipeline *p)
1358 {
1359         header_build_free(p);
1360
1361         for ( ; ; ) {
1362                 struct header *elem;
1363
1364                 elem = TAILQ_FIRST(&p->headers);
1365                 if (!elem)
1366                         break;
1367
1368                 TAILQ_REMOVE(&p->headers, elem, node);
1369                 free(elem);
1370         }
1371 }
1372
1373 /*
1374  * Meta-data.
1375  */
1376 static struct field *
1377 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1378 {
1379         if (!p->metadata_st)
1380                 return NULL;
1381
1382         if (name[0] != 'm' || name[1] != '.')
1383                 return NULL;
1384
1385         return struct_type_field_find(p->metadata_st, &name[2]);
1386 }
1387
1388 int
1389 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1390                                           const char *struct_type_name)
1391 {
1392         struct struct_type *st = NULL;
1393
1394         CHECK(p, EINVAL);
1395
1396         CHECK_NAME(struct_type_name, EINVAL);
1397         st  = struct_type_find(p, struct_type_name);
1398         CHECK(st, EINVAL);
1399         CHECK(!st->var_size, EINVAL);
1400         CHECK(!p->metadata_st, EINVAL);
1401
1402         p->metadata_st = st;
1403         p->metadata_struct_id = p->n_structs;
1404
1405         p->n_structs++;
1406
1407         return 0;
1408 }
1409
1410 static int
1411 metadata_build(struct rte_swx_pipeline *p)
1412 {
1413         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1414         uint32_t i;
1415
1416         /* Thread-level initialization. */
1417         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1418                 struct thread *t = &p->threads[i];
1419                 uint8_t *metadata;
1420
1421                 metadata = calloc(1, n_bytes);
1422                 CHECK(metadata, ENOMEM);
1423
1424                 t->metadata = metadata;
1425                 t->structs[p->metadata_struct_id] = metadata;
1426         }
1427
1428         return 0;
1429 }
1430
1431 static void
1432 metadata_build_free(struct rte_swx_pipeline *p)
1433 {
1434         uint32_t i;
1435
1436         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1437                 struct thread *t = &p->threads[i];
1438
1439                 free(t->metadata);
1440                 t->metadata = NULL;
1441         }
1442 }
1443
1444 static void
1445 metadata_free(struct rte_swx_pipeline *p)
1446 {
1447         metadata_build_free(p);
1448 }
1449
1450 /*
1451  * Instruction.
1452  */
1453 static int
1454 instruction_is_tx(enum instruction_type type)
1455 {
1456         switch (type) {
1457         case INSTR_TX:
1458         case INSTR_TX_I:
1459         case INSTR_DROP:
1460                 return 1;
1461
1462         default:
1463                 return 0;
1464         }
1465 }
1466
1467 static int
1468 instruction_does_tx(struct instruction *instr)
1469 {
1470         switch (instr->type) {
1471         case INSTR_TX:
1472         case INSTR_TX_I:
1473         case INSTR_DROP:
1474         case INSTR_HDR_EMIT_TX:
1475         case INSTR_HDR_EMIT2_TX:
1476         case INSTR_HDR_EMIT3_TX:
1477         case INSTR_HDR_EMIT4_TX:
1478         case INSTR_HDR_EMIT5_TX:
1479         case INSTR_HDR_EMIT6_TX:
1480         case INSTR_HDR_EMIT7_TX:
1481         case INSTR_HDR_EMIT8_TX:
1482                 return 1;
1483         default:
1484                 return 0;
1485         }
1486 }
1487
1488 static int
1489 instruction_is_jmp(struct instruction *instr)
1490 {
1491         switch (instr->type) {
1492         case INSTR_JMP:
1493         case INSTR_JMP_VALID:
1494         case INSTR_JMP_INVALID:
1495         case INSTR_JMP_HIT:
1496         case INSTR_JMP_MISS:
1497         case INSTR_JMP_ACTION_HIT:
1498         case INSTR_JMP_ACTION_MISS:
1499         case INSTR_JMP_EQ:
1500         case INSTR_JMP_EQ_MH:
1501         case INSTR_JMP_EQ_HM:
1502         case INSTR_JMP_EQ_HH:
1503         case INSTR_JMP_EQ_I:
1504         case INSTR_JMP_NEQ:
1505         case INSTR_JMP_NEQ_MH:
1506         case INSTR_JMP_NEQ_HM:
1507         case INSTR_JMP_NEQ_HH:
1508         case INSTR_JMP_NEQ_I:
1509         case INSTR_JMP_LT:
1510         case INSTR_JMP_LT_MH:
1511         case INSTR_JMP_LT_HM:
1512         case INSTR_JMP_LT_HH:
1513         case INSTR_JMP_LT_MI:
1514         case INSTR_JMP_LT_HI:
1515         case INSTR_JMP_GT:
1516         case INSTR_JMP_GT_MH:
1517         case INSTR_JMP_GT_HM:
1518         case INSTR_JMP_GT_HH:
1519         case INSTR_JMP_GT_MI:
1520         case INSTR_JMP_GT_HI:
1521                 return 1;
1522
1523         default:
1524                 return 0;
1525         }
1526 }
1527
1528 static int
1529 instruction_does_thread_yield(struct instruction *instr)
1530 {
1531         switch (instr->type) {
1532         case INSTR_RX:
1533         case INSTR_TABLE:
1534         case INSTR_TABLE_AF:
1535         case INSTR_SELECTOR:
1536         case INSTR_LEARNER:
1537         case INSTR_LEARNER_AF:
1538         case INSTR_EXTERN_OBJ:
1539         case INSTR_EXTERN_FUNC:
1540                 return 1;
1541         default:
1542                 return 0;
1543         }
1544 }
1545
1546 static struct field *
1547 action_field_parse(struct action *action, const char *name);
1548
1549 static struct field *
1550 struct_field_parse(struct rte_swx_pipeline *p,
1551                    struct action *action,
1552                    const char *name,
1553                    uint32_t *struct_id)
1554 {
1555         struct field *f;
1556
1557         switch (name[0]) {
1558         case 'h':
1559         {
1560                 struct header *header;
1561
1562                 f = header_field_parse(p, name, &header);
1563                 if (!f)
1564                         return NULL;
1565
1566                 *struct_id = header->struct_id;
1567                 return f;
1568         }
1569
1570         case 'm':
1571         {
1572                 f = metadata_field_parse(p, name);
1573                 if (!f)
1574                         return NULL;
1575
1576                 *struct_id = p->metadata_struct_id;
1577                 return f;
1578         }
1579
1580         case 't':
1581         {
1582                 if (!action)
1583                         return NULL;
1584
1585                 f = action_field_parse(action, name);
1586                 if (!f)
1587                         return NULL;
1588
1589                 *struct_id = 0;
1590                 return f;
1591         }
1592
1593         case 'e':
1594         {
1595                 struct extern_obj *obj;
1596
1597                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1598                 if (!f)
1599                         return NULL;
1600
1601                 *struct_id = obj->struct_id;
1602                 return f;
1603         }
1604
1605         case 'f':
1606         {
1607                 struct extern_func *func;
1608
1609                 f = extern_func_mailbox_field_parse(p, name, &func);
1610                 if (!f)
1611                         return NULL;
1612
1613                 *struct_id = func->struct_id;
1614                 return f;
1615         }
1616
1617         default:
1618                 return NULL;
1619         }
1620 }
1621
1622 /*
1623  * rx.
1624  */
1625 static int
1626 instr_rx_translate(struct rte_swx_pipeline *p,
1627                    struct action *action,
1628                    char **tokens,
1629                    int n_tokens,
1630                    struct instruction *instr,
1631                    struct instruction_data *data __rte_unused)
1632 {
1633         struct field *f;
1634
1635         CHECK(!action, EINVAL);
1636         CHECK(n_tokens == 2, EINVAL);
1637
1638         f = metadata_field_parse(p, tokens[1]);
1639         CHECK(f, EINVAL);
1640
1641         instr->type = INSTR_RX;
1642         instr->io.io.offset = f->offset / 8;
1643         instr->io.io.n_bits = f->n_bits;
1644         return 0;
1645 }
1646
1647 /*
1648  * tx.
1649  */
1650 static int
1651 instr_tx_translate(struct rte_swx_pipeline *p,
1652                    struct action *action __rte_unused,
1653                    char **tokens,
1654                    int n_tokens,
1655                    struct instruction *instr,
1656                    struct instruction_data *data __rte_unused)
1657 {
1658         char *port = tokens[1];
1659         struct field *f;
1660         uint32_t port_val;
1661
1662         CHECK(n_tokens == 2, EINVAL);
1663
1664         f = metadata_field_parse(p, port);
1665         if (f) {
1666                 instr->type = INSTR_TX;
1667                 instr->io.io.offset = f->offset / 8;
1668                 instr->io.io.n_bits = f->n_bits;
1669                 return 0;
1670         }
1671
1672         /* TX_I. */
1673         port_val = strtoul(port, &port, 0);
1674         CHECK(!port[0], EINVAL);
1675
1676         instr->type = INSTR_TX_I;
1677         instr->io.io.val = port_val;
1678         return 0;
1679 }
1680
1681 static int
1682 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1683                      struct action *action __rte_unused,
1684                      char **tokens __rte_unused,
1685                      int n_tokens,
1686                      struct instruction *instr,
1687                      struct instruction_data *data __rte_unused)
1688 {
1689         CHECK(n_tokens == 1, EINVAL);
1690
1691         /* DROP. */
1692         instr->type = INSTR_DROP;
1693         return 0;
1694 }
1695
1696 static inline void
1697 instr_tx_exec(struct rte_swx_pipeline *p)
1698 {
1699         struct thread *t = &p->threads[p->thread_id];
1700         struct instruction *ip = t->ip;
1701
1702         __instr_tx_exec(p, t, ip);
1703
1704         /* Thread. */
1705         thread_ip_reset(p, t);
1706         instr_rx_exec(p);
1707 }
1708
1709 static inline void
1710 instr_tx_i_exec(struct rte_swx_pipeline *p)
1711 {
1712         struct thread *t = &p->threads[p->thread_id];
1713         struct instruction *ip = t->ip;
1714
1715         __instr_tx_i_exec(p, t, ip);
1716
1717         /* Thread. */
1718         thread_ip_reset(p, t);
1719         instr_rx_exec(p);
1720 }
1721
1722 static inline void
1723 instr_drop_exec(struct rte_swx_pipeline *p)
1724 {
1725         struct thread *t = &p->threads[p->thread_id];
1726         struct instruction *ip = t->ip;
1727
1728         __instr_drop_exec(p, t, ip);
1729
1730         /* Thread. */
1731         thread_ip_reset(p, t);
1732         instr_rx_exec(p);
1733 }
1734
1735 /*
1736  * mirror.
1737  */
1738 static int
1739 instr_mirror_translate(struct rte_swx_pipeline *p,
1740                        struct action *action,
1741                        char **tokens,
1742                        int n_tokens,
1743                        struct instruction *instr,
1744                        struct instruction_data *data __rte_unused)
1745 {
1746         char *dst = tokens[1], *src = tokens[2];
1747         struct field *fdst, *fsrc;
1748         uint32_t dst_struct_id = 0, src_struct_id = 0;
1749
1750         CHECK(n_tokens == 3, EINVAL);
1751
1752         fdst = struct_field_parse(p, action, dst, &dst_struct_id);
1753         CHECK(fdst, EINVAL);
1754         CHECK(dst[0] != 'h', EINVAL);
1755         CHECK(!fdst->var_size, EINVAL);
1756
1757         fsrc = struct_field_parse(p, action, src, &src_struct_id);
1758         CHECK(fsrc, EINVAL);
1759         CHECK(src[0] != 'h', EINVAL);
1760         CHECK(!fsrc->var_size, EINVAL);
1761
1762         instr->type = INSTR_MIRROR;
1763         instr->mirror.dst.struct_id = (uint8_t)dst_struct_id;
1764         instr->mirror.dst.n_bits = fdst->n_bits;
1765         instr->mirror.dst.offset = fdst->offset / 8;
1766         instr->mirror.src.struct_id = (uint8_t)src_struct_id;
1767         instr->mirror.src.n_bits = fsrc->n_bits;
1768         instr->mirror.src.offset = fsrc->offset / 8;
1769
1770         return 0;
1771 }
1772
1773 static inline void
1774 instr_mirror_exec(struct rte_swx_pipeline *p)
1775 {
1776         struct thread *t = &p->threads[p->thread_id];
1777         struct instruction *ip = t->ip;
1778
1779         __instr_mirror_exec(p, t, ip);
1780
1781         /* Thread. */
1782         thread_ip_inc(p);
1783 }
1784
1785 /*
1786  * recirculate.
1787  */
1788 static int
1789 instr_recirculate_translate(struct rte_swx_pipeline *p __rte_unused,
1790                             struct action *action __rte_unused,
1791                             char **tokens __rte_unused,
1792                             int n_tokens,
1793                             struct instruction *instr,
1794                             struct instruction_data *data __rte_unused)
1795 {
1796         CHECK(n_tokens == 1, EINVAL);
1797
1798         instr->type = INSTR_RECIRCULATE;
1799         return 0;
1800 }
1801
1802 static int
1803 instr_recircid_translate(struct rte_swx_pipeline *p,
1804                          struct action *action __rte_unused,
1805                          char **tokens,
1806                          int n_tokens,
1807                          struct instruction *instr,
1808                          struct instruction_data *data __rte_unused)
1809 {
1810         struct field *f;
1811
1812         CHECK(n_tokens == 2, EINVAL);
1813
1814         f = metadata_field_parse(p, tokens[1]);
1815         CHECK(f, EINVAL);
1816
1817         instr->type = INSTR_RECIRCID;
1818         instr->io.io.offset = f->offset / 8;
1819         instr->io.io.n_bits = f->n_bits;
1820         return 0;
1821 }
1822
1823 static inline void
1824 instr_recirculate_exec(struct rte_swx_pipeline *p)
1825 {
1826         struct thread *t = &p->threads[p->thread_id];
1827         struct instruction *ip = t->ip;
1828
1829         __instr_recirculate_exec(p, t, ip);
1830
1831         /* Thread. */
1832         thread_ip_inc(p);
1833 }
1834
1835 static inline void
1836 instr_recircid_exec(struct rte_swx_pipeline *p)
1837 {
1838         struct thread *t = &p->threads[p->thread_id];
1839         struct instruction *ip = t->ip;
1840
1841         __instr_recircid_exec(p, t, ip);
1842
1843         /* Thread. */
1844         thread_ip_inc(p);
1845 }
1846
1847 /*
1848  * extract.
1849  */
1850 static int
1851 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1852                             struct action *action,
1853                             char **tokens,
1854                             int n_tokens,
1855                             struct instruction *instr,
1856                             struct instruction_data *data __rte_unused)
1857 {
1858         struct header *h;
1859
1860         CHECK(!action, EINVAL);
1861         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1862
1863         h = header_parse(p, tokens[1]);
1864         CHECK(h, EINVAL);
1865
1866         if (n_tokens == 2) {
1867                 CHECK(!h->st->var_size, EINVAL);
1868
1869                 instr->type = INSTR_HDR_EXTRACT;
1870                 instr->io.hdr.header_id[0] = h->id;
1871                 instr->io.hdr.struct_id[0] = h->struct_id;
1872                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1873         } else {
1874                 struct field *mf;
1875
1876                 CHECK(h->st->var_size, EINVAL);
1877
1878                 mf = metadata_field_parse(p, tokens[2]);
1879                 CHECK(mf, EINVAL);
1880                 CHECK(!mf->var_size, EINVAL);
1881
1882                 instr->type = INSTR_HDR_EXTRACT_M;
1883                 instr->io.io.offset = mf->offset / 8;
1884                 instr->io.io.n_bits = mf->n_bits;
1885                 instr->io.hdr.header_id[0] = h->id;
1886                 instr->io.hdr.struct_id[0] = h->struct_id;
1887                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1888         }
1889
1890         return 0;
1891 }
1892
1893 static int
1894 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1895                               struct action *action,
1896                               char **tokens,
1897                               int n_tokens,
1898                               struct instruction *instr,
1899                               struct instruction_data *data __rte_unused)
1900 {
1901         struct header *h;
1902
1903         CHECK(!action, EINVAL);
1904         CHECK(n_tokens == 2, EINVAL);
1905
1906         h = header_parse(p, tokens[1]);
1907         CHECK(h, EINVAL);
1908         CHECK(!h->st->var_size, EINVAL);
1909
1910         instr->type = INSTR_HDR_LOOKAHEAD;
1911         instr->io.hdr.header_id[0] = h->id;
1912         instr->io.hdr.struct_id[0] = h->struct_id;
1913         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1914
1915         return 0;
1916 }
1917
1918 static inline void
1919 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1920 {
1921         struct thread *t = &p->threads[p->thread_id];
1922         struct instruction *ip = t->ip;
1923
1924         __instr_hdr_extract_exec(p, t, ip);
1925
1926         /* Thread. */
1927         thread_ip_inc(p);
1928 }
1929
1930 static inline void
1931 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1932 {
1933         struct thread *t = &p->threads[p->thread_id];
1934         struct instruction *ip = t->ip;
1935
1936         __instr_hdr_extract2_exec(p, t, ip);
1937
1938         /* Thread. */
1939         thread_ip_inc(p);
1940 }
1941
1942 static inline void
1943 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1944 {
1945         struct thread *t = &p->threads[p->thread_id];
1946         struct instruction *ip = t->ip;
1947
1948         __instr_hdr_extract3_exec(p, t, ip);
1949
1950         /* Thread. */
1951         thread_ip_inc(p);
1952 }
1953
1954 static inline void
1955 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1956 {
1957         struct thread *t = &p->threads[p->thread_id];
1958         struct instruction *ip = t->ip;
1959
1960         __instr_hdr_extract4_exec(p, t, ip);
1961
1962         /* Thread. */
1963         thread_ip_inc(p);
1964 }
1965
1966 static inline void
1967 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1968 {
1969         struct thread *t = &p->threads[p->thread_id];
1970         struct instruction *ip = t->ip;
1971
1972         __instr_hdr_extract5_exec(p, t, ip);
1973
1974         /* Thread. */
1975         thread_ip_inc(p);
1976 }
1977
1978 static inline void
1979 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1980 {
1981         struct thread *t = &p->threads[p->thread_id];
1982         struct instruction *ip = t->ip;
1983
1984         __instr_hdr_extract6_exec(p, t, ip);
1985
1986         /* Thread. */
1987         thread_ip_inc(p);
1988 }
1989
1990 static inline void
1991 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1992 {
1993         struct thread *t = &p->threads[p->thread_id];
1994         struct instruction *ip = t->ip;
1995
1996         __instr_hdr_extract7_exec(p, t, ip);
1997
1998         /* Thread. */
1999         thread_ip_inc(p);
2000 }
2001
2002 static inline void
2003 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2004 {
2005         struct thread *t = &p->threads[p->thread_id];
2006         struct instruction *ip = t->ip;
2007
2008         __instr_hdr_extract8_exec(p, t, ip);
2009
2010         /* Thread. */
2011         thread_ip_inc(p);
2012 }
2013
2014 static inline void
2015 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
2016 {
2017         struct thread *t = &p->threads[p->thread_id];
2018         struct instruction *ip = t->ip;
2019
2020         __instr_hdr_extract_m_exec(p, t, ip);
2021
2022         /* Thread. */
2023         thread_ip_inc(p);
2024 }
2025
2026 static inline void
2027 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
2028 {
2029         struct thread *t = &p->threads[p->thread_id];
2030         struct instruction *ip = t->ip;
2031
2032         __instr_hdr_lookahead_exec(p, t, ip);
2033
2034         /* Thread. */
2035         thread_ip_inc(p);
2036 }
2037
2038 /*
2039  * emit.
2040  */
2041 static int
2042 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2043                          struct action *action __rte_unused,
2044                          char **tokens,
2045                          int n_tokens,
2046                          struct instruction *instr,
2047                          struct instruction_data *data __rte_unused)
2048 {
2049         struct header *h;
2050
2051         CHECK(n_tokens == 2, EINVAL);
2052
2053         h = header_parse(p, tokens[1]);
2054         CHECK(h, EINVAL);
2055
2056         instr->type = INSTR_HDR_EMIT;
2057         instr->io.hdr.header_id[0] = h->id;
2058         instr->io.hdr.struct_id[0] = h->struct_id;
2059         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2060         return 0;
2061 }
2062
2063 static inline void
2064 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2065 {
2066         struct thread *t = &p->threads[p->thread_id];
2067         struct instruction *ip = t->ip;
2068
2069         __instr_hdr_emit_exec(p, t, ip);
2070
2071         /* Thread. */
2072         thread_ip_inc(p);
2073 }
2074
2075 static inline void
2076 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2077 {
2078         struct thread *t = &p->threads[p->thread_id];
2079         struct instruction *ip = t->ip;
2080
2081         __instr_hdr_emit_tx_exec(p, t, ip);
2082
2083         /* Thread. */
2084         thread_ip_reset(p, t);
2085         instr_rx_exec(p);
2086 }
2087
2088 static inline void
2089 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2090 {
2091         struct thread *t = &p->threads[p->thread_id];
2092         struct instruction *ip = t->ip;
2093
2094         __instr_hdr_emit2_tx_exec(p, t, ip);
2095
2096         /* Thread. */
2097         thread_ip_reset(p, t);
2098         instr_rx_exec(p);
2099 }
2100
2101 static inline void
2102 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2103 {
2104         struct thread *t = &p->threads[p->thread_id];
2105         struct instruction *ip = t->ip;
2106
2107         __instr_hdr_emit3_tx_exec(p, t, ip);
2108
2109         /* Thread. */
2110         thread_ip_reset(p, t);
2111         instr_rx_exec(p);
2112 }
2113
2114 static inline void
2115 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2116 {
2117         struct thread *t = &p->threads[p->thread_id];
2118         struct instruction *ip = t->ip;
2119
2120         __instr_hdr_emit4_tx_exec(p, t, ip);
2121
2122         /* Thread. */
2123         thread_ip_reset(p, t);
2124         instr_rx_exec(p);
2125 }
2126
2127 static inline void
2128 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2129 {
2130         struct thread *t = &p->threads[p->thread_id];
2131         struct instruction *ip = t->ip;
2132
2133         __instr_hdr_emit5_tx_exec(p, t, ip);
2134
2135         /* Thread. */
2136         thread_ip_reset(p, t);
2137         instr_rx_exec(p);
2138 }
2139
2140 static inline void
2141 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2142 {
2143         struct thread *t = &p->threads[p->thread_id];
2144         struct instruction *ip = t->ip;
2145
2146         __instr_hdr_emit6_tx_exec(p, t, ip);
2147
2148         /* Thread. */
2149         thread_ip_reset(p, t);
2150         instr_rx_exec(p);
2151 }
2152
2153 static inline void
2154 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2155 {
2156         struct thread *t = &p->threads[p->thread_id];
2157         struct instruction *ip = t->ip;
2158
2159         __instr_hdr_emit7_tx_exec(p, t, ip);
2160
2161         /* Thread. */
2162         thread_ip_reset(p, t);
2163         instr_rx_exec(p);
2164 }
2165
2166 static inline void
2167 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2168 {
2169         struct thread *t = &p->threads[p->thread_id];
2170         struct instruction *ip = t->ip;
2171
2172         __instr_hdr_emit8_tx_exec(p, t, ip);
2173
2174         /* Thread. */
2175         thread_ip_reset(p, t);
2176         instr_rx_exec(p);
2177 }
2178
2179 /*
2180  * validate.
2181  */
2182 static int
2183 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2184                              struct action *action __rte_unused,
2185                              char **tokens,
2186                              int n_tokens,
2187                              struct instruction *instr,
2188                              struct instruction_data *data __rte_unused)
2189 {
2190         struct header *h;
2191
2192         CHECK(n_tokens == 2, EINVAL);
2193
2194         h = header_parse(p, tokens[1]);
2195         CHECK(h, EINVAL);
2196
2197         instr->type = INSTR_HDR_VALIDATE;
2198         instr->valid.header_id = h->id;
2199         return 0;
2200 }
2201
2202 static inline void
2203 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2204 {
2205         struct thread *t = &p->threads[p->thread_id];
2206         struct instruction *ip = t->ip;
2207
2208         __instr_hdr_validate_exec(p, t, ip);
2209
2210         /* Thread. */
2211         thread_ip_inc(p);
2212 }
2213
2214 /*
2215  * invalidate.
2216  */
2217 static int
2218 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2219                                struct action *action __rte_unused,
2220                                char **tokens,
2221                                int n_tokens,
2222                                struct instruction *instr,
2223                                struct instruction_data *data __rte_unused)
2224 {
2225         struct header *h;
2226
2227         CHECK(n_tokens == 2, EINVAL);
2228
2229         h = header_parse(p, tokens[1]);
2230         CHECK(h, EINVAL);
2231
2232         instr->type = INSTR_HDR_INVALIDATE;
2233         instr->valid.header_id = h->id;
2234         return 0;
2235 }
2236
2237 static inline void
2238 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2239 {
2240         struct thread *t = &p->threads[p->thread_id];
2241         struct instruction *ip = t->ip;
2242
2243         __instr_hdr_invalidate_exec(p, t, ip);
2244
2245         /* Thread. */
2246         thread_ip_inc(p);
2247 }
2248
2249 /*
2250  * table.
2251  */
2252 static struct table *
2253 table_find(struct rte_swx_pipeline *p, const char *name);
2254
2255 static struct selector *
2256 selector_find(struct rte_swx_pipeline *p, const char *name);
2257
2258 static struct learner *
2259 learner_find(struct rte_swx_pipeline *p, const char *name);
2260
2261 static int
2262 instr_table_translate(struct rte_swx_pipeline *p,
2263                       struct action *action,
2264                       char **tokens,
2265                       int n_tokens,
2266                       struct instruction *instr,
2267                       struct instruction_data *data __rte_unused)
2268 {
2269         struct table *t;
2270         struct selector *s;
2271         struct learner *l;
2272
2273         CHECK(!action, EINVAL);
2274         CHECK(n_tokens == 2, EINVAL);
2275
2276         t = table_find(p, tokens[1]);
2277         if (t) {
2278                 instr->type = INSTR_TABLE;
2279                 instr->table.table_id = t->id;
2280                 return 0;
2281         }
2282
2283         s = selector_find(p, tokens[1]);
2284         if (s) {
2285                 instr->type = INSTR_SELECTOR;
2286                 instr->table.table_id = s->id;
2287                 return 0;
2288         }
2289
2290         l = learner_find(p, tokens[1]);
2291         if (l) {
2292                 instr->type = INSTR_LEARNER;
2293                 instr->table.table_id = l->id;
2294                 return 0;
2295         }
2296
2297         CHECK(0, EINVAL);
2298 }
2299
2300 static inline void
2301 instr_table_exec(struct rte_swx_pipeline *p)
2302 {
2303         struct thread *t = &p->threads[p->thread_id];
2304         struct instruction *ip = t->ip;
2305         uint32_t table_id = ip->table.table_id;
2306         struct rte_swx_table_state *ts = &t->table_state[table_id];
2307         struct table_runtime *table = &t->tables[table_id];
2308         struct table_statistics *stats = &p->table_stats[table_id];
2309         uint64_t action_id, n_pkts_hit, n_pkts_action;
2310         uint8_t *action_data;
2311         int done, hit;
2312
2313         /* Table. */
2314         done = table->func(ts->obj,
2315                            table->mailbox,
2316                            table->key,
2317                            &action_id,
2318                            &action_data,
2319                            &hit);
2320         if (!done) {
2321                 /* Thread. */
2322                 TRACE("[Thread %2u] table %u (not finalized)\n",
2323                       p->thread_id,
2324                       table_id);
2325
2326                 thread_yield(p);
2327                 return;
2328         }
2329
2330         action_id = hit ? action_id : ts->default_action_id;
2331         action_data = hit ? action_data : ts->default_action_data;
2332         n_pkts_hit = stats->n_pkts_hit[hit];
2333         n_pkts_action = stats->n_pkts_action[action_id];
2334
2335         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2336               p->thread_id,
2337               table_id,
2338               hit ? "hit" : "miss",
2339               (uint32_t)action_id);
2340
2341         t->action_id = action_id;
2342         t->structs[0] = action_data;
2343         t->hit = hit;
2344         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2345         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2346
2347         /* Thread. */
2348         thread_ip_action_call(p, t, action_id);
2349 }
2350
2351 static inline void
2352 instr_table_af_exec(struct rte_swx_pipeline *p)
2353 {
2354         struct thread *t = &p->threads[p->thread_id];
2355         struct instruction *ip = t->ip;
2356         uint32_t table_id = ip->table.table_id;
2357         struct rte_swx_table_state *ts = &t->table_state[table_id];
2358         struct table_runtime *table = &t->tables[table_id];
2359         struct table_statistics *stats = &p->table_stats[table_id];
2360         uint64_t action_id, n_pkts_hit, n_pkts_action;
2361         uint8_t *action_data;
2362         action_func_t action_func;
2363         int done, hit;
2364
2365         /* Table. */
2366         done = table->func(ts->obj,
2367                            table->mailbox,
2368                            table->key,
2369                            &action_id,
2370                            &action_data,
2371                            &hit);
2372         if (!done) {
2373                 /* Thread. */
2374                 TRACE("[Thread %2u] table %u (not finalized)\n",
2375                       p->thread_id,
2376                       table_id);
2377
2378                 thread_yield(p);
2379                 return;
2380         }
2381
2382         action_id = hit ? action_id : ts->default_action_id;
2383         action_data = hit ? action_data : ts->default_action_data;
2384         action_func = p->action_funcs[action_id];
2385         n_pkts_hit = stats->n_pkts_hit[hit];
2386         n_pkts_action = stats->n_pkts_action[action_id];
2387
2388         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2389               p->thread_id,
2390               table_id,
2391               hit ? "hit" : "miss",
2392               (uint32_t)action_id);
2393
2394         t->action_id = action_id;
2395         t->structs[0] = action_data;
2396         t->hit = hit;
2397         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2398         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2399
2400         /* Thread. */
2401         thread_ip_inc(p);
2402
2403         /* Action. */
2404         action_func(p);
2405 }
2406
2407 static inline void
2408 instr_selector_exec(struct rte_swx_pipeline *p)
2409 {
2410         struct thread *t = &p->threads[p->thread_id];
2411         struct instruction *ip = t->ip;
2412         uint32_t selector_id = ip->table.table_id;
2413         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2414         struct selector_runtime *selector = &t->selectors[selector_id];
2415         struct selector_statistics *stats = &p->selector_stats[selector_id];
2416         uint64_t n_pkts = stats->n_pkts;
2417         int done;
2418
2419         /* Table. */
2420         done = rte_swx_table_selector_select(ts->obj,
2421                            selector->mailbox,
2422                            selector->group_id_buffer,
2423                            selector->selector_buffer,
2424                            selector->member_id_buffer);
2425         if (!done) {
2426                 /* Thread. */
2427                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2428                       p->thread_id,
2429                       selector_id);
2430
2431                 thread_yield(p);
2432                 return;
2433         }
2434
2435
2436         TRACE("[Thread %2u] selector %u\n",
2437               p->thread_id,
2438               selector_id);
2439
2440         stats->n_pkts = n_pkts + 1;
2441
2442         /* Thread. */
2443         thread_ip_inc(p);
2444 }
2445
2446 static inline void
2447 instr_learner_exec(struct rte_swx_pipeline *p)
2448 {
2449         struct thread *t = &p->threads[p->thread_id];
2450         struct instruction *ip = t->ip;
2451         uint32_t learner_id = ip->table.table_id;
2452         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2453                 p->n_selectors + learner_id];
2454         struct learner_runtime *l = &t->learners[learner_id];
2455         struct learner_statistics *stats = &p->learner_stats[learner_id];
2456         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2457         uint8_t *action_data;
2458         int done, hit;
2459
2460         /* Table. */
2461         time = rte_get_tsc_cycles();
2462
2463         done = rte_swx_table_learner_lookup(ts->obj,
2464                                             l->mailbox,
2465                                             time,
2466                                             l->key,
2467                                             &action_id,
2468                                             &action_data,
2469                                             &hit);
2470         if (!done) {
2471                 /* Thread. */
2472                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2473                       p->thread_id,
2474                       learner_id);
2475
2476                 thread_yield(p);
2477                 return;
2478         }
2479
2480         action_id = hit ? action_id : ts->default_action_id;
2481         action_data = hit ? action_data : ts->default_action_data;
2482         n_pkts_hit = stats->n_pkts_hit[hit];
2483         n_pkts_action = stats->n_pkts_action[action_id];
2484
2485         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2486               p->thread_id,
2487               learner_id,
2488               hit ? "hit" : "miss",
2489               (uint32_t)action_id);
2490
2491         t->action_id = action_id;
2492         t->structs[0] = action_data;
2493         t->hit = hit;
2494         t->learner_id = learner_id;
2495         t->time = time;
2496         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2497         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2498
2499         /* Thread. */
2500         thread_ip_action_call(p, t, action_id);
2501 }
2502
2503 static inline void
2504 instr_learner_af_exec(struct rte_swx_pipeline *p)
2505 {
2506         struct thread *t = &p->threads[p->thread_id];
2507         struct instruction *ip = t->ip;
2508         uint32_t learner_id = ip->table.table_id;
2509         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2510                 p->n_selectors + learner_id];
2511         struct learner_runtime *l = &t->learners[learner_id];
2512         struct learner_statistics *stats = &p->learner_stats[learner_id];
2513         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2514         uint8_t *action_data;
2515         action_func_t action_func;
2516         int done, hit;
2517
2518         /* Table. */
2519         time = rte_get_tsc_cycles();
2520
2521         done = rte_swx_table_learner_lookup(ts->obj,
2522                                             l->mailbox,
2523                                             time,
2524                                             l->key,
2525                                             &action_id,
2526                                             &action_data,
2527                                             &hit);
2528         if (!done) {
2529                 /* Thread. */
2530                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2531                       p->thread_id,
2532                       learner_id);
2533
2534                 thread_yield(p);
2535                 return;
2536         }
2537
2538         action_id = hit ? action_id : ts->default_action_id;
2539         action_data = hit ? action_data : ts->default_action_data;
2540         action_func = p->action_funcs[action_id];
2541         n_pkts_hit = stats->n_pkts_hit[hit];
2542         n_pkts_action = stats->n_pkts_action[action_id];
2543
2544         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2545               p->thread_id,
2546               learner_id,
2547               hit ? "hit" : "miss",
2548               (uint32_t)action_id);
2549
2550         t->action_id = action_id;
2551         t->structs[0] = action_data;
2552         t->hit = hit;
2553         t->learner_id = learner_id;
2554         t->time = time;
2555         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2556         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2557
2558         /* Thread. */
2559         thread_ip_inc(p);
2560
2561         /* Action */
2562         action_func(p);
2563 }
2564
2565 /*
2566  * learn.
2567  */
2568 static struct action *
2569 action_find(struct rte_swx_pipeline *p, const char *name);
2570
2571 static int
2572 action_has_nbo_args(struct action *a);
2573
2574 static int
2575 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2576
2577 static int
2578 instr_learn_translate(struct rte_swx_pipeline *p,
2579                       struct action *action,
2580                       char **tokens,
2581                       int n_tokens,
2582                       struct instruction *instr,
2583                       struct instruction_data *data __rte_unused)
2584 {
2585         struct action *a;
2586         struct field *mf_first_arg = NULL, *mf_timeout_id = NULL;
2587         const char *mf_first_arg_name, *mf_timeout_id_name;
2588
2589         CHECK(action, EINVAL);
2590         CHECK((n_tokens == 3) || (n_tokens == 4), EINVAL);
2591
2592         /* Action. */
2593         a = action_find(p, tokens[1]);
2594         CHECK(a, EINVAL);
2595         CHECK(!action_has_nbo_args(a), EINVAL);
2596
2597         /* Action first argument. */
2598         mf_first_arg_name = (n_tokens == 4) ? tokens[2] : NULL;
2599         CHECK(!learner_action_args_check(p, a, mf_first_arg_name), EINVAL);
2600
2601         if (mf_first_arg_name) {
2602                 mf_first_arg = metadata_field_parse(p, mf_first_arg_name);
2603                 CHECK(mf_first_arg, EINVAL);
2604         }
2605
2606         /* Timeout ID. */
2607         mf_timeout_id_name = (n_tokens == 4) ? tokens[3] : tokens[2];
2608         CHECK_NAME(mf_timeout_id_name, EINVAL);
2609         mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
2610         CHECK(mf_timeout_id, EINVAL);
2611
2612         /* Instruction. */
2613         instr->type = INSTR_LEARNER_LEARN;
2614         instr->learn.action_id = a->id;
2615         instr->learn.mf_first_arg_offset = mf_first_arg ? (mf_first_arg->offset / 8) : 0;
2616         instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
2617         instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;
2618
2619         return 0;
2620 }
2621
2622 static inline void
2623 instr_learn_exec(struct rte_swx_pipeline *p)
2624 {
2625         struct thread *t = &p->threads[p->thread_id];
2626         struct instruction *ip = t->ip;
2627
2628         __instr_learn_exec(p, t, ip);
2629
2630         /* Thread. */
2631         thread_ip_inc(p);
2632 }
2633
2634 /*
2635  * rearm.
2636  */
2637 static int
2638 instr_rearm_translate(struct rte_swx_pipeline *p,
2639                       struct action *action,
2640                       char **tokens,
2641                       int n_tokens,
2642                       struct instruction *instr,
2643                       struct instruction_data *data __rte_unused)
2644 {
2645         struct field *mf_timeout_id;
2646         const char *mf_timeout_id_name;
2647
2648         CHECK(action, EINVAL);
2649         CHECK((n_tokens == 1) || (n_tokens == 2), EINVAL);
2650
2651         /* INSTR_LEARNER_REARM. */
2652         if (n_tokens == 1) {
2653                 instr->type = INSTR_LEARNER_REARM;
2654                 return 0;
2655         }
2656
2657         /* INSTR_LEARNER_REARM_NEW. */
2658         mf_timeout_id_name = tokens[1];
2659         CHECK_NAME(mf_timeout_id_name, EINVAL);
2660         mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
2661         CHECK(mf_timeout_id, EINVAL);
2662
2663         instr->type = INSTR_LEARNER_REARM_NEW;
2664         instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
2665         instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;
2666
2667         return 0;
2668 }
2669
2670 static inline void
2671 instr_rearm_exec(struct rte_swx_pipeline *p)
2672 {
2673         struct thread *t = &p->threads[p->thread_id];
2674         struct instruction *ip = t->ip;
2675
2676         __instr_rearm_exec(p, t, ip);
2677
2678         /* Thread. */
2679         thread_ip_inc(p);
2680 }
2681
2682 static inline void
2683 instr_rearm_new_exec(struct rte_swx_pipeline *p)
2684 {
2685         struct thread *t = &p->threads[p->thread_id];
2686         struct instruction *ip = t->ip;
2687
2688         __instr_rearm_new_exec(p, t, ip);
2689
2690         /* Thread. */
2691         thread_ip_inc(p);
2692 }
2693
2694 /*
2695  * forget.
2696  */
2697 static int
2698 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2699                        struct action *action,
2700                        char **tokens __rte_unused,
2701                        int n_tokens,
2702                        struct instruction *instr,
2703                        struct instruction_data *data __rte_unused)
2704 {
2705         CHECK(action, EINVAL);
2706         CHECK(n_tokens == 1, EINVAL);
2707
2708         instr->type = INSTR_LEARNER_FORGET;
2709
2710         return 0;
2711 }
2712
2713 static inline void
2714 instr_forget_exec(struct rte_swx_pipeline *p)
2715 {
2716         struct thread *t = &p->threads[p->thread_id];
2717         struct instruction *ip = t->ip;
2718
2719         __instr_forget_exec(p, t, ip);
2720
2721         /* Thread. */
2722         thread_ip_inc(p);
2723 }
2724
2725 /*
2726  * extern.
2727  */
2728 static int
2729 instr_extern_translate(struct rte_swx_pipeline *p,
2730                        struct action *action __rte_unused,
2731                        char **tokens,
2732                        int n_tokens,
2733                        struct instruction *instr,
2734                        struct instruction_data *data __rte_unused)
2735 {
2736         char *token = tokens[1];
2737
2738         CHECK(n_tokens == 2, EINVAL);
2739
2740         if (token[0] == 'e') {
2741                 struct extern_obj *obj;
2742                 struct extern_type_member_func *func;
2743
2744                 func = extern_obj_member_func_parse(p, token, &obj);
2745                 CHECK(func, EINVAL);
2746
2747                 instr->type = INSTR_EXTERN_OBJ;
2748                 instr->ext_obj.ext_obj_id = obj->id;
2749                 instr->ext_obj.func_id = func->id;
2750
2751                 return 0;
2752         }
2753
2754         if (token[0] == 'f') {
2755                 struct extern_func *func;
2756
2757                 func = extern_func_parse(p, token);
2758                 CHECK(func, EINVAL);
2759
2760                 instr->type = INSTR_EXTERN_FUNC;
2761                 instr->ext_func.ext_func_id = func->id;
2762
2763                 return 0;
2764         }
2765
2766         CHECK(0, EINVAL);
2767 }
2768
2769 static inline void
2770 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2771 {
2772         struct thread *t = &p->threads[p->thread_id];
2773         struct instruction *ip = t->ip;
2774         uint32_t done;
2775
2776         /* Extern object member function execute. */
2777         done = __instr_extern_obj_exec(p, t, ip);
2778
2779         /* Thread. */
2780         thread_ip_inc_cond(t, done);
2781         thread_yield_cond(p, done ^ 1);
2782 }
2783
2784 static inline void
2785 instr_extern_func_exec(struct rte_swx_pipeline *p)
2786 {
2787         struct thread *t = &p->threads[p->thread_id];
2788         struct instruction *ip = t->ip;
2789         uint32_t done;
2790
2791         /* Extern function execute. */
2792         done = __instr_extern_func_exec(p, t, ip);
2793
2794         /* Thread. */
2795         thread_ip_inc_cond(t, done);
2796         thread_yield_cond(p, done ^ 1);
2797 }
2798
2799 /*
2800  * mov.
2801  */
2802 static int
2803 instr_mov_translate(struct rte_swx_pipeline *p,
2804                     struct action *action,
2805                     char **tokens,
2806                     int n_tokens,
2807                     struct instruction *instr,
2808                     struct instruction_data *data __rte_unused)
2809 {
2810         char *dst = tokens[1], *src = tokens[2];
2811         struct field *fdst, *fsrc;
2812         uint64_t src_val;
2813         uint32_t dst_struct_id = 0, src_struct_id = 0;
2814
2815         CHECK(n_tokens == 3, EINVAL);
2816
2817         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2818         CHECK(fdst, EINVAL);
2819         CHECK(!fdst->var_size, EINVAL);
2820
2821         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2822         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2823         if (fsrc) {
2824                 CHECK(!fsrc->var_size, EINVAL);
2825
2826                 instr->type = INSTR_MOV;
2827                 if (dst[0] != 'h' && src[0] == 'h')
2828                         instr->type = INSTR_MOV_MH;
2829                 if (dst[0] == 'h' && src[0] != 'h')
2830                         instr->type = INSTR_MOV_HM;
2831                 if (dst[0] == 'h' && src[0] == 'h')
2832                         instr->type = INSTR_MOV_HH;
2833
2834                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2835                 instr->mov.dst.n_bits = fdst->n_bits;
2836                 instr->mov.dst.offset = fdst->offset / 8;
2837                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2838                 instr->mov.src.n_bits = fsrc->n_bits;
2839                 instr->mov.src.offset = fsrc->offset / 8;
2840                 return 0;
2841         }
2842
2843         /* MOV_I. */
2844         src_val = strtoull(src, &src, 0);
2845         CHECK(!src[0], EINVAL);
2846
2847         if (dst[0] == 'h')
2848                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2849
2850         instr->type = INSTR_MOV_I;
2851         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2852         instr->mov.dst.n_bits = fdst->n_bits;
2853         instr->mov.dst.offset = fdst->offset / 8;
2854         instr->mov.src_val = src_val;
2855         return 0;
2856 }
2857
2858 static inline void
2859 instr_mov_exec(struct rte_swx_pipeline *p)
2860 {
2861         struct thread *t = &p->threads[p->thread_id];
2862         struct instruction *ip = t->ip;
2863
2864         __instr_mov_exec(p, t, ip);
2865
2866         /* Thread. */
2867         thread_ip_inc(p);
2868 }
2869
2870 static inline void
2871 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2872 {
2873         struct thread *t = &p->threads[p->thread_id];
2874         struct instruction *ip = t->ip;
2875
2876         __instr_mov_mh_exec(p, t, ip);
2877
2878         /* Thread. */
2879         thread_ip_inc(p);
2880 }
2881
2882 static inline void
2883 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2884 {
2885         struct thread *t = &p->threads[p->thread_id];
2886         struct instruction *ip = t->ip;
2887
2888         __instr_mov_hm_exec(p, t, ip);
2889
2890         /* Thread. */
2891         thread_ip_inc(p);
2892 }
2893
2894 static inline void
2895 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2896 {
2897         struct thread *t = &p->threads[p->thread_id];
2898         struct instruction *ip = t->ip;
2899
2900         __instr_mov_hh_exec(p, t, ip);
2901
2902         /* Thread. */
2903         thread_ip_inc(p);
2904 }
2905
2906 static inline void
2907 instr_mov_i_exec(struct rte_swx_pipeline *p)
2908 {
2909         struct thread *t = &p->threads[p->thread_id];
2910         struct instruction *ip = t->ip;
2911
2912         __instr_mov_i_exec(p, t, ip);
2913
2914         /* Thread. */
2915         thread_ip_inc(p);
2916 }
2917
2918 /*
2919  * dma.
2920  */
2921 static inline void
2922 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2923 {
2924         struct thread *t = &p->threads[p->thread_id];
2925         struct instruction *ip = t->ip;
2926
2927         __instr_dma_ht_exec(p, t, ip);
2928
2929         /* Thread. */
2930         thread_ip_inc(p);
2931 }
2932
2933 static inline void
2934 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2935 {
2936         struct thread *t = &p->threads[p->thread_id];
2937         struct instruction *ip = t->ip;
2938
2939         __instr_dma_ht2_exec(p, t, ip);
2940
2941         /* Thread. */
2942         thread_ip_inc(p);
2943 }
2944
2945 static inline void
2946 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2947 {
2948         struct thread *t = &p->threads[p->thread_id];
2949         struct instruction *ip = t->ip;
2950
2951         __instr_dma_ht3_exec(p, t, ip);
2952
2953         /* Thread. */
2954         thread_ip_inc(p);
2955 }
2956
2957 static inline void
2958 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2959 {
2960         struct thread *t = &p->threads[p->thread_id];
2961         struct instruction *ip = t->ip;
2962
2963         __instr_dma_ht4_exec(p, t, ip);
2964
2965         /* Thread. */
2966         thread_ip_inc(p);
2967 }
2968
2969 static inline void
2970 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2971 {
2972         struct thread *t = &p->threads[p->thread_id];
2973         struct instruction *ip = t->ip;
2974
2975         __instr_dma_ht5_exec(p, t, ip);
2976
2977         /* Thread. */
2978         thread_ip_inc(p);
2979 }
2980
2981 static inline void
2982 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2983 {
2984         struct thread *t = &p->threads[p->thread_id];
2985         struct instruction *ip = t->ip;
2986
2987         __instr_dma_ht6_exec(p, t, ip);
2988
2989         /* Thread. */
2990         thread_ip_inc(p);
2991 }
2992
2993 static inline void
2994 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2995 {
2996         struct thread *t = &p->threads[p->thread_id];
2997         struct instruction *ip = t->ip;
2998
2999         __instr_dma_ht7_exec(p, t, ip);
3000
3001         /* Thread. */
3002         thread_ip_inc(p);
3003 }
3004
3005 static inline void
3006 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3007 {
3008         struct thread *t = &p->threads[p->thread_id];
3009         struct instruction *ip = t->ip;
3010
3011         __instr_dma_ht8_exec(p, t, ip);
3012
3013         /* Thread. */
3014         thread_ip_inc(p);
3015 }
3016
3017 /*
3018  * alu.
3019  */
3020 static int
3021 instr_alu_add_translate(struct rte_swx_pipeline *p,
3022                         struct action *action,
3023                         char **tokens,
3024                         int n_tokens,
3025                         struct instruction *instr,
3026                         struct instruction_data *data __rte_unused)
3027 {
3028         char *dst = tokens[1], *src = tokens[2];
3029         struct field *fdst, *fsrc;
3030         uint64_t src_val;
3031         uint32_t dst_struct_id = 0, src_struct_id = 0;
3032
3033         CHECK(n_tokens == 3, EINVAL);
3034
3035         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3036         CHECK(fdst, EINVAL);
3037         CHECK(!fdst->var_size, EINVAL);
3038
3039         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3040         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3041         if (fsrc) {
3042                 CHECK(!fsrc->var_size, EINVAL);
3043
3044                 instr->type = INSTR_ALU_ADD;
3045                 if (dst[0] == 'h' && src[0] != 'h')
3046                         instr->type = INSTR_ALU_ADD_HM;
3047                 if (dst[0] != 'h' && src[0] == 'h')
3048                         instr->type = INSTR_ALU_ADD_MH;
3049                 if (dst[0] == 'h' && src[0] == 'h')
3050                         instr->type = INSTR_ALU_ADD_HH;
3051
3052                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3053                 instr->alu.dst.n_bits = fdst->n_bits;
3054                 instr->alu.dst.offset = fdst->offset / 8;
3055                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3056                 instr->alu.src.n_bits = fsrc->n_bits;
3057                 instr->alu.src.offset = fsrc->offset / 8;
3058                 return 0;
3059         }
3060
3061         /* ADD_MI, ADD_HI. */
3062         src_val = strtoull(src, &src, 0);
3063         CHECK(!src[0], EINVAL);
3064
3065         instr->type = INSTR_ALU_ADD_MI;
3066         if (dst[0] == 'h')
3067                 instr->type = INSTR_ALU_ADD_HI;
3068
3069         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3070         instr->alu.dst.n_bits = fdst->n_bits;
3071         instr->alu.dst.offset = fdst->offset / 8;
3072         instr->alu.src_val = src_val;
3073         return 0;
3074 }
3075
3076 static int
3077 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3078                         struct action *action,
3079                         char **tokens,
3080                         int n_tokens,
3081                         struct instruction *instr,
3082                         struct instruction_data *data __rte_unused)
3083 {
3084         char *dst = tokens[1], *src = tokens[2];
3085         struct field *fdst, *fsrc;
3086         uint64_t src_val;
3087         uint32_t dst_struct_id = 0, src_struct_id = 0;
3088
3089         CHECK(n_tokens == 3, EINVAL);
3090
3091         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3092         CHECK(fdst, EINVAL);
3093         CHECK(!fdst->var_size, EINVAL);
3094
3095         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3096         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3097         if (fsrc) {
3098                 CHECK(!fsrc->var_size, EINVAL);
3099
3100                 instr->type = INSTR_ALU_SUB;
3101                 if (dst[0] == 'h' && src[0] != 'h')
3102                         instr->type = INSTR_ALU_SUB_HM;
3103                 if (dst[0] != 'h' && src[0] == 'h')
3104                         instr->type = INSTR_ALU_SUB_MH;
3105                 if (dst[0] == 'h' && src[0] == 'h')
3106                         instr->type = INSTR_ALU_SUB_HH;
3107
3108                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3109                 instr->alu.dst.n_bits = fdst->n_bits;
3110                 instr->alu.dst.offset = fdst->offset / 8;
3111                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3112                 instr->alu.src.n_bits = fsrc->n_bits;
3113                 instr->alu.src.offset = fsrc->offset / 8;
3114                 return 0;
3115         }
3116
3117         /* SUB_MI, SUB_HI. */
3118         src_val = strtoull(src, &src, 0);
3119         CHECK(!src[0], EINVAL);
3120
3121         instr->type = INSTR_ALU_SUB_MI;
3122         if (dst[0] == 'h')
3123                 instr->type = INSTR_ALU_SUB_HI;
3124
3125         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3126         instr->alu.dst.n_bits = fdst->n_bits;
3127         instr->alu.dst.offset = fdst->offset / 8;
3128         instr->alu.src_val = src_val;
3129         return 0;
3130 }
3131
3132 static int
3133 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3134                           struct action *action __rte_unused,
3135                           char **tokens,
3136                           int n_tokens,
3137                           struct instruction *instr,
3138                           struct instruction_data *data __rte_unused)
3139 {
3140         char *dst = tokens[1], *src = tokens[2];
3141         struct header *hdst, *hsrc;
3142         struct field *fdst, *fsrc;
3143
3144         CHECK(n_tokens == 3, EINVAL);
3145
3146         fdst = header_field_parse(p, dst, &hdst);
3147         CHECK(fdst, EINVAL);
3148         CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3149
3150         /* CKADD_FIELD. */
3151         fsrc = header_field_parse(p, src, &hsrc);
3152         if (fsrc) {
3153                 CHECK(!fsrc->var_size, EINVAL);
3154
3155                 instr->type = INSTR_ALU_CKADD_FIELD;
3156                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3157                 instr->alu.dst.n_bits = fdst->n_bits;
3158                 instr->alu.dst.offset = fdst->offset / 8;
3159                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3160                 instr->alu.src.n_bits = fsrc->n_bits;
3161                 instr->alu.src.offset = fsrc->offset / 8;
3162                 return 0;
3163         }
3164
3165         /* CKADD_STRUCT, CKADD_STRUCT20. */
3166         hsrc = header_parse(p, src);
3167         CHECK(hsrc, EINVAL);
3168
3169         instr->type = INSTR_ALU_CKADD_STRUCT;
3170         if (!hsrc->st->var_size && ((hsrc->st->n_bits / 8) == 20))
3171                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3172
3173         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3174         instr->alu.dst.n_bits = fdst->n_bits;
3175         instr->alu.dst.offset = fdst->offset / 8;
3176         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3177         instr->alu.src.n_bits = (uint8_t)hsrc->id; /* The src header ID is stored here. */
3178         instr->alu.src.offset = 0; /* Unused. */
3179         return 0;
3180 }
3181
3182 static int
3183 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3184                           struct action *action __rte_unused,
3185                           char **tokens,
3186                           int n_tokens,
3187                           struct instruction *instr,
3188                           struct instruction_data *data __rte_unused)
3189 {
3190         char *dst = tokens[1], *src = tokens[2];
3191         struct header *hdst, *hsrc;
3192         struct field *fdst, *fsrc;
3193
3194         CHECK(n_tokens == 3, EINVAL);
3195
3196         fdst = header_field_parse(p, dst, &hdst);
3197         CHECK(fdst, EINVAL);
3198         CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3199
3200         fsrc = header_field_parse(p, src, &hsrc);
3201         CHECK(fsrc, EINVAL);
3202         CHECK(!fsrc->var_size, EINVAL);
3203
3204         instr->type = INSTR_ALU_CKSUB_FIELD;
3205         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3206         instr->alu.dst.n_bits = fdst->n_bits;
3207         instr->alu.dst.offset = fdst->offset / 8;
3208         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3209         instr->alu.src.n_bits = fsrc->n_bits;
3210         instr->alu.src.offset = fsrc->offset / 8;
3211         return 0;
3212 }
3213
3214 static int
3215 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3216                         struct action *action,
3217                         char **tokens,
3218                         int n_tokens,
3219                         struct instruction *instr,
3220                         struct instruction_data *data __rte_unused)
3221 {
3222         char *dst = tokens[1], *src = tokens[2];
3223         struct field *fdst, *fsrc;
3224         uint64_t src_val;
3225         uint32_t dst_struct_id = 0, src_struct_id = 0;
3226
3227         CHECK(n_tokens == 3, EINVAL);
3228
3229         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3230         CHECK(fdst, EINVAL);
3231         CHECK(!fdst->var_size, EINVAL);
3232
3233         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3234         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3235         if (fsrc) {
3236                 CHECK(!fsrc->var_size, EINVAL);
3237
3238                 instr->type = INSTR_ALU_SHL;
3239                 if (dst[0] == 'h' && src[0] != 'h')
3240                         instr->type = INSTR_ALU_SHL_HM;
3241                 if (dst[0] != 'h' && src[0] == 'h')
3242                         instr->type = INSTR_ALU_SHL_MH;
3243                 if (dst[0] == 'h' && src[0] == 'h')
3244                         instr->type = INSTR_ALU_SHL_HH;
3245
3246                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3247                 instr->alu.dst.n_bits = fdst->n_bits;
3248                 instr->alu.dst.offset = fdst->offset / 8;
3249                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3250                 instr->alu.src.n_bits = fsrc->n_bits;
3251                 instr->alu.src.offset = fsrc->offset / 8;
3252                 return 0;
3253         }
3254
3255         /* SHL_MI, SHL_HI. */
3256         src_val = strtoull(src, &src, 0);
3257         CHECK(!src[0], EINVAL);
3258
3259         instr->type = INSTR_ALU_SHL_MI;
3260         if (dst[0] == 'h')
3261                 instr->type = INSTR_ALU_SHL_HI;
3262
3263         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3264         instr->alu.dst.n_bits = fdst->n_bits;
3265         instr->alu.dst.offset = fdst->offset / 8;
3266         instr->alu.src_val = src_val;
3267         return 0;
3268 }
3269
3270 static int
3271 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3272                         struct action *action,
3273                         char **tokens,
3274                         int n_tokens,
3275                         struct instruction *instr,
3276                         struct instruction_data *data __rte_unused)
3277 {
3278         char *dst = tokens[1], *src = tokens[2];
3279         struct field *fdst, *fsrc;
3280         uint64_t src_val;
3281         uint32_t dst_struct_id = 0, src_struct_id = 0;
3282
3283         CHECK(n_tokens == 3, EINVAL);
3284
3285         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3286         CHECK(fdst, EINVAL);
3287         CHECK(!fdst->var_size, EINVAL);
3288
3289         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3290         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3291         if (fsrc) {
3292                 CHECK(!fsrc->var_size, EINVAL);
3293
3294                 instr->type = INSTR_ALU_SHR;
3295                 if (dst[0] == 'h' && src[0] != 'h')
3296                         instr->type = INSTR_ALU_SHR_HM;
3297                 if (dst[0] != 'h' && src[0] == 'h')
3298                         instr->type = INSTR_ALU_SHR_MH;
3299                 if (dst[0] == 'h' && src[0] == 'h')
3300                         instr->type = INSTR_ALU_SHR_HH;
3301
3302                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3303                 instr->alu.dst.n_bits = fdst->n_bits;
3304                 instr->alu.dst.offset = fdst->offset / 8;
3305                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3306                 instr->alu.src.n_bits = fsrc->n_bits;
3307                 instr->alu.src.offset = fsrc->offset / 8;
3308                 return 0;
3309         }
3310
3311         /* SHR_MI, SHR_HI. */
3312         src_val = strtoull(src, &src, 0);
3313         CHECK(!src[0], EINVAL);
3314
3315         instr->type = INSTR_ALU_SHR_MI;
3316         if (dst[0] == 'h')
3317                 instr->type = INSTR_ALU_SHR_HI;
3318
3319         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3320         instr->alu.dst.n_bits = fdst->n_bits;
3321         instr->alu.dst.offset = fdst->offset / 8;
3322         instr->alu.src_val = src_val;
3323         return 0;
3324 }
3325
3326 static int
3327 instr_alu_and_translate(struct rte_swx_pipeline *p,
3328                         struct action *action,
3329                         char **tokens,
3330                         int n_tokens,
3331                         struct instruction *instr,
3332                         struct instruction_data *data __rte_unused)
3333 {
3334         char *dst = tokens[1], *src = tokens[2];
3335         struct field *fdst, *fsrc;
3336         uint64_t src_val;
3337         uint32_t dst_struct_id = 0, src_struct_id = 0;
3338
3339         CHECK(n_tokens == 3, EINVAL);
3340
3341         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3342         CHECK(fdst, EINVAL);
3343         CHECK(!fdst->var_size, EINVAL);
3344
3345         /* AND, AND_MH, AND_HM, AND_HH. */
3346         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3347         if (fsrc) {
3348                 CHECK(!fsrc->var_size, EINVAL);
3349
3350                 instr->type = INSTR_ALU_AND;
3351                 if (dst[0] != 'h' && src[0] == 'h')
3352                         instr->type = INSTR_ALU_AND_MH;
3353                 if (dst[0] == 'h' && src[0] != 'h')
3354                         instr->type = INSTR_ALU_AND_HM;
3355                 if (dst[0] == 'h' && src[0] == 'h')
3356                         instr->type = INSTR_ALU_AND_HH;
3357
3358                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3359                 instr->alu.dst.n_bits = fdst->n_bits;
3360                 instr->alu.dst.offset = fdst->offset / 8;
3361                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3362                 instr->alu.src.n_bits = fsrc->n_bits;
3363                 instr->alu.src.offset = fsrc->offset / 8;
3364                 return 0;
3365         }
3366
3367         /* AND_I. */
3368         src_val = strtoull(src, &src, 0);
3369         CHECK(!src[0], EINVAL);
3370
3371         if (dst[0] == 'h')
3372                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3373
3374         instr->type = INSTR_ALU_AND_I;
3375         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3376         instr->alu.dst.n_bits = fdst->n_bits;
3377         instr->alu.dst.offset = fdst->offset / 8;
3378         instr->alu.src_val = src_val;
3379         return 0;
3380 }
3381
3382 static int
3383 instr_alu_or_translate(struct rte_swx_pipeline *p,
3384                        struct action *action,
3385                        char **tokens,
3386                        int n_tokens,
3387                        struct instruction *instr,
3388                        struct instruction_data *data __rte_unused)
3389 {
3390         char *dst = tokens[1], *src = tokens[2];
3391         struct field *fdst, *fsrc;
3392         uint64_t src_val;
3393         uint32_t dst_struct_id = 0, src_struct_id = 0;
3394
3395         CHECK(n_tokens == 3, EINVAL);
3396
3397         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3398         CHECK(fdst, EINVAL);
3399         CHECK(!fdst->var_size, EINVAL);
3400
3401         /* OR, OR_MH, OR_HM, OR_HH. */
3402         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3403         if (fsrc) {
3404                 CHECK(!fsrc->var_size, EINVAL);
3405
3406                 instr->type = INSTR_ALU_OR;
3407                 if (dst[0] != 'h' && src[0] == 'h')
3408                         instr->type = INSTR_ALU_OR_MH;
3409                 if (dst[0] == 'h' && src[0] != 'h')
3410                         instr->type = INSTR_ALU_OR_HM;
3411                 if (dst[0] == 'h' && src[0] == 'h')
3412                         instr->type = INSTR_ALU_OR_HH;
3413
3414                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3415                 instr->alu.dst.n_bits = fdst->n_bits;
3416                 instr->alu.dst.offset = fdst->offset / 8;
3417                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3418                 instr->alu.src.n_bits = fsrc->n_bits;
3419                 instr->alu.src.offset = fsrc->offset / 8;
3420                 return 0;
3421         }
3422
3423         /* OR_I. */
3424         src_val = strtoull(src, &src, 0);
3425         CHECK(!src[0], EINVAL);
3426
3427         if (dst[0] == 'h')
3428                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3429
3430         instr->type = INSTR_ALU_OR_I;
3431         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3432         instr->alu.dst.n_bits = fdst->n_bits;
3433         instr->alu.dst.offset = fdst->offset / 8;
3434         instr->alu.src_val = src_val;
3435         return 0;
3436 }
3437
3438 static int
3439 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3440                         struct action *action,
3441                         char **tokens,
3442                         int n_tokens,
3443                         struct instruction *instr,
3444                         struct instruction_data *data __rte_unused)
3445 {
3446         char *dst = tokens[1], *src = tokens[2];
3447         struct field *fdst, *fsrc;
3448         uint64_t src_val;
3449         uint32_t dst_struct_id = 0, src_struct_id = 0;
3450
3451         CHECK(n_tokens == 3, EINVAL);
3452
3453         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3454         CHECK(fdst, EINVAL);
3455         CHECK(!fdst->var_size, EINVAL);
3456
3457         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3458         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3459         if (fsrc) {
3460                 CHECK(!fsrc->var_size, EINVAL);
3461
3462                 instr->type = INSTR_ALU_XOR;
3463                 if (dst[0] != 'h' && src[0] == 'h')
3464                         instr->type = INSTR_ALU_XOR_MH;
3465                 if (dst[0] == 'h' && src[0] != 'h')
3466                         instr->type = INSTR_ALU_XOR_HM;
3467                 if (dst[0] == 'h' && src[0] == 'h')
3468                         instr->type = INSTR_ALU_XOR_HH;
3469
3470                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3471                 instr->alu.dst.n_bits = fdst->n_bits;
3472                 instr->alu.dst.offset = fdst->offset / 8;
3473                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3474                 instr->alu.src.n_bits = fsrc->n_bits;
3475                 instr->alu.src.offset = fsrc->offset / 8;
3476                 return 0;
3477         }
3478
3479         /* XOR_I. */
3480         src_val = strtoull(src, &src, 0);
3481         CHECK(!src[0], EINVAL);
3482
3483         if (dst[0] == 'h')
3484                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3485
3486         instr->type = INSTR_ALU_XOR_I;
3487         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3488         instr->alu.dst.n_bits = fdst->n_bits;
3489         instr->alu.dst.offset = fdst->offset / 8;
3490         instr->alu.src_val = src_val;
3491         return 0;
3492 }
3493
3494 static inline void
3495 instr_alu_add_exec(struct rte_swx_pipeline *p)
3496 {
3497         struct thread *t = &p->threads[p->thread_id];
3498         struct instruction *ip = t->ip;
3499
3500         /* Structs */
3501         __instr_alu_add_exec(p, t, ip);
3502
3503         /* Thread. */
3504         thread_ip_inc(p);
3505 }
3506
3507 static inline void
3508 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3509 {
3510         struct thread *t = &p->threads[p->thread_id];
3511         struct instruction *ip = t->ip;
3512
3513         /* Structs. */
3514         __instr_alu_add_mh_exec(p, t, ip);
3515
3516         /* Thread. */
3517         thread_ip_inc(p);
3518 }
3519
3520 static inline void
3521 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3522 {
3523         struct thread *t = &p->threads[p->thread_id];
3524         struct instruction *ip = t->ip;
3525
3526         /* Structs. */
3527         __instr_alu_add_hm_exec(p, t, ip);
3528
3529         /* Thread. */
3530         thread_ip_inc(p);
3531 }
3532
3533 static inline void
3534 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3535 {
3536         struct thread *t = &p->threads[p->thread_id];
3537         struct instruction *ip = t->ip;
3538
3539         /* Structs. */
3540         __instr_alu_add_hh_exec(p, t, ip);
3541
3542         /* Thread. */
3543         thread_ip_inc(p);
3544 }
3545
3546 static inline void
3547 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3548 {
3549         struct thread *t = &p->threads[p->thread_id];
3550         struct instruction *ip = t->ip;
3551
3552         /* Structs. */
3553         __instr_alu_add_mi_exec(p, t, ip);
3554
3555         /* Thread. */
3556         thread_ip_inc(p);
3557 }
3558
3559 static inline void
3560 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3561 {
3562         struct thread *t = &p->threads[p->thread_id];
3563         struct instruction *ip = t->ip;
3564
3565         /* Structs. */
3566         __instr_alu_add_hi_exec(p, t, ip);
3567
3568         /* Thread. */
3569         thread_ip_inc(p);
3570 }
3571
3572 static inline void
3573 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3574 {
3575         struct thread *t = &p->threads[p->thread_id];
3576         struct instruction *ip = t->ip;
3577
3578         /* Structs. */
3579         __instr_alu_sub_exec(p, t, ip);
3580
3581         /* Thread. */
3582         thread_ip_inc(p);
3583 }
3584
3585 static inline void
3586 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3587 {
3588         struct thread *t = &p->threads[p->thread_id];
3589         struct instruction *ip = t->ip;
3590
3591         /* Structs. */
3592         __instr_alu_sub_mh_exec(p, t, ip);
3593
3594         /* Thread. */
3595         thread_ip_inc(p);
3596 }
3597
3598 static inline void
3599 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3600 {
3601         struct thread *t = &p->threads[p->thread_id];
3602         struct instruction *ip = t->ip;
3603
3604         /* Structs. */
3605         __instr_alu_sub_hm_exec(p, t, ip);
3606
3607         /* Thread. */
3608         thread_ip_inc(p);
3609 }
3610
3611 static inline void
3612 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3613 {
3614         struct thread *t = &p->threads[p->thread_id];
3615         struct instruction *ip = t->ip;
3616
3617         /* Structs. */
3618         __instr_alu_sub_hh_exec(p, t, ip);
3619
3620         /* Thread. */
3621         thread_ip_inc(p);
3622 }
3623
3624 static inline void
3625 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3626 {
3627         struct thread *t = &p->threads[p->thread_id];
3628         struct instruction *ip = t->ip;
3629
3630         /* Structs. */
3631         __instr_alu_sub_mi_exec(p, t, ip);
3632
3633         /* Thread. */
3634         thread_ip_inc(p);
3635 }
3636
3637 static inline void
3638 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3639 {
3640         struct thread *t = &p->threads[p->thread_id];
3641         struct instruction *ip = t->ip;
3642
3643         /* Structs. */
3644         __instr_alu_sub_hi_exec(p, t, ip);
3645
3646         /* Thread. */
3647         thread_ip_inc(p);
3648 }
3649
3650 static inline void
3651 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3652 {
3653         struct thread *t = &p->threads[p->thread_id];
3654         struct instruction *ip = t->ip;
3655
3656         /* Structs. */
3657         __instr_alu_shl_exec(p, t, ip);
3658
3659         /* Thread. */
3660         thread_ip_inc(p);
3661 }
3662
3663 static inline void
3664 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3665 {
3666         struct thread *t = &p->threads[p->thread_id];
3667         struct instruction *ip = t->ip;
3668
3669         /* Structs. */
3670         __instr_alu_shl_mh_exec(p, t, ip);
3671
3672         /* Thread. */
3673         thread_ip_inc(p);
3674 }
3675
3676 static inline void
3677 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3678 {
3679         struct thread *t = &p->threads[p->thread_id];
3680         struct instruction *ip = t->ip;
3681
3682         /* Structs. */
3683         __instr_alu_shl_hm_exec(p, t, ip);
3684
3685         /* Thread. */
3686         thread_ip_inc(p);
3687 }
3688
3689 static inline void
3690 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3691 {
3692         struct thread *t = &p->threads[p->thread_id];
3693         struct instruction *ip = t->ip;
3694
3695         /* Structs. */
3696         __instr_alu_shl_hh_exec(p, t, ip);
3697
3698         /* Thread. */
3699         thread_ip_inc(p);
3700 }
3701
3702 static inline void
3703 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3704 {
3705         struct thread *t = &p->threads[p->thread_id];
3706         struct instruction *ip = t->ip;
3707
3708         /* Structs. */
3709         __instr_alu_shl_mi_exec(p, t, ip);
3710
3711         /* Thread. */
3712         thread_ip_inc(p);
3713 }
3714
3715 static inline void
3716 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3717 {
3718         struct thread *t = &p->threads[p->thread_id];
3719         struct instruction *ip = t->ip;
3720
3721         /* Structs. */
3722         __instr_alu_shl_hi_exec(p, t, ip);
3723
3724         /* Thread. */
3725         thread_ip_inc(p);
3726 }
3727
3728 static inline void
3729 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3730 {
3731         struct thread *t = &p->threads[p->thread_id];
3732         struct instruction *ip = t->ip;
3733
3734         /* Structs. */
3735         __instr_alu_shr_exec(p, t, ip);
3736
3737         /* Thread. */
3738         thread_ip_inc(p);
3739 }
3740
3741 static inline void
3742 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3743 {
3744         struct thread *t = &p->threads[p->thread_id];
3745         struct instruction *ip = t->ip;
3746
3747         /* Structs. */
3748         __instr_alu_shr_mh_exec(p, t, ip);
3749
3750         /* Thread. */
3751         thread_ip_inc(p);
3752 }
3753
3754 static inline void
3755 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3756 {
3757         struct thread *t = &p->threads[p->thread_id];
3758         struct instruction *ip = t->ip;
3759
3760         /* Structs. */
3761         __instr_alu_shr_hm_exec(p, t, ip);
3762
3763         /* Thread. */
3764         thread_ip_inc(p);
3765 }
3766
3767 static inline void
3768 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3769 {
3770         struct thread *t = &p->threads[p->thread_id];
3771         struct instruction *ip = t->ip;
3772
3773         /* Structs. */
3774         __instr_alu_shr_hh_exec(p, t, ip);
3775
3776         /* Thread. */
3777         thread_ip_inc(p);
3778 }
3779
3780 static inline void
3781 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3782 {
3783         struct thread *t = &p->threads[p->thread_id];
3784         struct instruction *ip = t->ip;
3785
3786         /* Structs. */
3787         __instr_alu_shr_mi_exec(p, t, ip);
3788
3789         /* Thread. */
3790         thread_ip_inc(p);
3791 }
3792
3793 static inline void
3794 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3795 {
3796         struct thread *t = &p->threads[p->thread_id];
3797         struct instruction *ip = t->ip;
3798
3799         /* Structs. */
3800         __instr_alu_shr_hi_exec(p, t, ip);
3801
3802         /* Thread. */
3803         thread_ip_inc(p);
3804 }
3805
3806 static inline void
3807 instr_alu_and_exec(struct rte_swx_pipeline *p)
3808 {
3809         struct thread *t = &p->threads[p->thread_id];
3810         struct instruction *ip = t->ip;
3811
3812         /* Structs. */
3813         __instr_alu_and_exec(p, t, ip);
3814
3815         /* Thread. */
3816         thread_ip_inc(p);
3817 }
3818
3819 static inline void
3820 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3821 {
3822         struct thread *t = &p->threads[p->thread_id];
3823         struct instruction *ip = t->ip;
3824
3825         /* Structs. */
3826         __instr_alu_and_mh_exec(p, t, ip);
3827
3828         /* Thread. */
3829         thread_ip_inc(p);
3830 }
3831
3832 static inline void
3833 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3834 {
3835         struct thread *t = &p->threads[p->thread_id];
3836         struct instruction *ip = t->ip;
3837
3838         /* Structs. */
3839         __instr_alu_and_hm_exec(p, t, ip);
3840
3841         /* Thread. */
3842         thread_ip_inc(p);
3843 }
3844
3845 static inline void
3846 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3847 {
3848         struct thread *t = &p->threads[p->thread_id];
3849         struct instruction *ip = t->ip;
3850
3851         /* Structs. */
3852         __instr_alu_and_hh_exec(p, t, ip);
3853
3854         /* Thread. */
3855         thread_ip_inc(p);
3856 }
3857
3858 static inline void
3859 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3860 {
3861         struct thread *t = &p->threads[p->thread_id];
3862         struct instruction *ip = t->ip;
3863
3864         /* Structs. */
3865         __instr_alu_and_i_exec(p, t, ip);
3866
3867         /* Thread. */
3868         thread_ip_inc(p);
3869 }
3870
3871 static inline void
3872 instr_alu_or_exec(struct rte_swx_pipeline *p)
3873 {
3874         struct thread *t = &p->threads[p->thread_id];
3875         struct instruction *ip = t->ip;
3876
3877         /* Structs. */
3878         __instr_alu_or_exec(p, t, ip);
3879
3880         /* Thread. */
3881         thread_ip_inc(p);
3882 }
3883
3884 static inline void
3885 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3886 {
3887         struct thread *t = &p->threads[p->thread_id];
3888         struct instruction *ip = t->ip;
3889
3890         /* Structs. */
3891         __instr_alu_or_mh_exec(p, t, ip);
3892
3893         /* Thread. */
3894         thread_ip_inc(p);
3895 }
3896
3897 static inline void
3898 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3899 {
3900         struct thread *t = &p->threads[p->thread_id];
3901         struct instruction *ip = t->ip;
3902
3903         /* Structs. */
3904         __instr_alu_or_hm_exec(p, t, ip);
3905
3906         /* Thread. */
3907         thread_ip_inc(p);
3908 }
3909
3910 static inline void
3911 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3912 {
3913         struct thread *t = &p->threads[p->thread_id];
3914         struct instruction *ip = t->ip;
3915
3916         /* Structs. */
3917         __instr_alu_or_hh_exec(p, t, ip);
3918
3919         /* Thread. */
3920         thread_ip_inc(p);
3921 }
3922
3923 static inline void
3924 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3925 {
3926         struct thread *t = &p->threads[p->thread_id];
3927         struct instruction *ip = t->ip;
3928
3929         /* Structs. */
3930         __instr_alu_or_i_exec(p, t, ip);
3931
3932         /* Thread. */
3933         thread_ip_inc(p);
3934 }
3935
3936 static inline void
3937 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3938 {
3939         struct thread *t = &p->threads[p->thread_id];
3940         struct instruction *ip = t->ip;
3941
3942         /* Structs. */
3943         __instr_alu_xor_exec(p, t, ip);
3944
3945         /* Thread. */
3946         thread_ip_inc(p);
3947 }
3948
3949 static inline void
3950 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3951 {
3952         struct thread *t = &p->threads[p->thread_id];
3953         struct instruction *ip = t->ip;
3954
3955         /* Structs. */
3956         __instr_alu_xor_mh_exec(p, t, ip);
3957
3958         /* Thread. */
3959         thread_ip_inc(p);
3960 }
3961
3962 static inline void
3963 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3964 {
3965         struct thread *t = &p->threads[p->thread_id];
3966         struct instruction *ip = t->ip;
3967
3968         /* Structs. */
3969         __instr_alu_xor_hm_exec(p, t, ip);
3970
3971         /* Thread. */
3972         thread_ip_inc(p);
3973 }
3974
3975 static inline void
3976 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3977 {
3978         struct thread *t = &p->threads[p->thread_id];
3979         struct instruction *ip = t->ip;
3980
3981         /* Structs. */
3982         __instr_alu_xor_hh_exec(p, t, ip);
3983
3984         /* Thread. */
3985         thread_ip_inc(p);
3986 }
3987
3988 static inline void
3989 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3990 {
3991         struct thread *t = &p->threads[p->thread_id];
3992         struct instruction *ip = t->ip;
3993
3994         /* Structs. */
3995         __instr_alu_xor_i_exec(p, t, ip);
3996
3997         /* Thread. */
3998         thread_ip_inc(p);
3999 }
4000
4001 static inline void
4002 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4003 {
4004         struct thread *t = &p->threads[p->thread_id];
4005         struct instruction *ip = t->ip;
4006
4007         /* Structs. */
4008         __instr_alu_ckadd_field_exec(p, t, ip);
4009
4010         /* Thread. */
4011         thread_ip_inc(p);
4012 }
4013
4014 static inline void
4015 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4016 {
4017         struct thread *t = &p->threads[p->thread_id];
4018         struct instruction *ip = t->ip;
4019
4020         /* Structs. */
4021         __instr_alu_cksub_field_exec(p, t, ip);
4022
4023         /* Thread. */
4024         thread_ip_inc(p);
4025 }
4026
4027 static inline void
4028 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4029 {
4030         struct thread *t = &p->threads[p->thread_id];
4031         struct instruction *ip = t->ip;
4032
4033         /* Structs. */
4034         __instr_alu_ckadd_struct20_exec(p, t, ip);
4035
4036         /* Thread. */
4037         thread_ip_inc(p);
4038 }
4039
4040 static inline void
4041 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4042 {
4043         struct thread *t = &p->threads[p->thread_id];
4044         struct instruction *ip = t->ip;
4045
4046         /* Structs. */
4047         __instr_alu_ckadd_struct_exec(p, t, ip);
4048
4049         /* Thread. */
4050         thread_ip_inc(p);
4051 }
4052
4053 /*
4054  * Register array.
4055  */
4056 static struct regarray *
4057 regarray_find(struct rte_swx_pipeline *p, const char *name);
4058
4059 static int
4060 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4061                       struct action *action,
4062                       char **tokens,
4063                       int n_tokens,
4064                       struct instruction *instr,
4065                       struct instruction_data *data __rte_unused)
4066 {
4067         char *regarray = tokens[1], *idx = tokens[2];
4068         struct regarray *r;
4069         struct field *fidx;
4070         uint32_t idx_struct_id, idx_val;
4071
4072         CHECK(n_tokens == 3, EINVAL);
4073
4074         r = regarray_find(p, regarray);
4075         CHECK(r, EINVAL);
4076
4077         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4078         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4079         if (fidx) {
4080                 CHECK(!fidx->var_size, EINVAL);
4081
4082                 instr->type = INSTR_REGPREFETCH_RM;
4083                 if (idx[0] == 'h')
4084                         instr->type = INSTR_REGPREFETCH_RH;
4085
4086                 instr->regarray.regarray_id = r->id;
4087                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4088                 instr->regarray.idx.n_bits = fidx->n_bits;
4089                 instr->regarray.idx.offset = fidx->offset / 8;
4090                 instr->regarray.dstsrc_val = 0; /* Unused. */
4091                 return 0;
4092         }
4093
4094         /* REGPREFETCH_RI. */
4095         idx_val = strtoul(idx, &idx, 0);
4096         CHECK(!idx[0], EINVAL);
4097
4098         instr->type = INSTR_REGPREFETCH_RI;
4099         instr->regarray.regarray_id = r->id;
4100         instr->regarray.idx_val = idx_val;
4101         instr->regarray.dstsrc_val = 0; /* Unused. */
4102         return 0;
4103 }
4104
4105 static int
4106 instr_regrd_translate(struct rte_swx_pipeline *p,
4107                       struct action *action,
4108                       char **tokens,
4109                       int n_tokens,
4110                       struct instruction *instr,
4111                       struct instruction_data *data __rte_unused)
4112 {
4113         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4114         struct regarray *r;
4115         struct field *fdst, *fidx;
4116         uint32_t dst_struct_id, idx_struct_id, idx_val;
4117
4118         CHECK(n_tokens == 4, EINVAL);
4119
4120         r = regarray_find(p, regarray);
4121         CHECK(r, EINVAL);
4122
4123         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4124         CHECK(fdst, EINVAL);
4125         CHECK(!fdst->var_size, EINVAL);
4126
4127         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4128         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4129         if (fidx) {
4130                 CHECK(!fidx->var_size, EINVAL);
4131
4132                 instr->type = INSTR_REGRD_MRM;
4133                 if (dst[0] == 'h' && idx[0] != 'h')
4134                         instr->type = INSTR_REGRD_HRM;
4135                 if (dst[0] != 'h' && idx[0] == 'h')
4136                         instr->type = INSTR_REGRD_MRH;
4137                 if (dst[0] == 'h' && idx[0] == 'h')
4138                         instr->type = INSTR_REGRD_HRH;
4139
4140                 instr->regarray.regarray_id = r->id;
4141                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4142                 instr->regarray.idx.n_bits = fidx->n_bits;
4143                 instr->regarray.idx.offset = fidx->offset / 8;
4144                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4145                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4146                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4147                 return 0;
4148         }
4149
4150         /* REGRD_MRI, REGRD_HRI. */
4151         idx_val = strtoul(idx, &idx, 0);
4152         CHECK(!idx[0], EINVAL);
4153
4154         instr->type = INSTR_REGRD_MRI;
4155         if (dst[0] == 'h')
4156                 instr->type = INSTR_REGRD_HRI;
4157
4158         instr->regarray.regarray_id = r->id;
4159         instr->regarray.idx_val = idx_val;
4160         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4161         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4162         instr->regarray.dstsrc.offset = fdst->offset / 8;
4163         return 0;
4164 }
4165
4166 static int
4167 instr_regwr_translate(struct rte_swx_pipeline *p,
4168                       struct action *action,
4169                       char **tokens,
4170                       int n_tokens,
4171                       struct instruction *instr,
4172                       struct instruction_data *data __rte_unused)
4173 {
4174         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4175         struct regarray *r;
4176         struct field *fidx, *fsrc;
4177         uint64_t src_val;
4178         uint32_t idx_struct_id, idx_val, src_struct_id;
4179
4180         CHECK(n_tokens == 4, EINVAL);
4181
4182         r = regarray_find(p, regarray);
4183         CHECK(r, EINVAL);
4184
4185         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4186         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4187         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4188         if (fidx && fsrc) {
4189                 CHECK(!fidx->var_size, EINVAL);
4190                 CHECK(!fsrc->var_size, EINVAL);
4191
4192                 instr->type = INSTR_REGWR_RMM;
4193                 if (idx[0] == 'h' && src[0] != 'h')
4194                         instr->type = INSTR_REGWR_RHM;
4195                 if (idx[0] != 'h' && src[0] == 'h')
4196                         instr->type = INSTR_REGWR_RMH;
4197                 if (idx[0] == 'h' && src[0] == 'h')
4198                         instr->type = INSTR_REGWR_RHH;
4199
4200                 instr->regarray.regarray_id = r->id;
4201                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4202                 instr->regarray.idx.n_bits = fidx->n_bits;
4203                 instr->regarray.idx.offset = fidx->offset / 8;
4204                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4205                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4206                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4207                 return 0;
4208         }
4209
4210         /* REGWR_RHI, REGWR_RMI. */
4211         if (fidx && !fsrc) {
4212                 CHECK(!fidx->var_size, EINVAL);
4213
4214                 src_val = strtoull(src, &src, 0);
4215                 CHECK(!src[0], EINVAL);
4216
4217                 instr->type = INSTR_REGWR_RMI;
4218                 if (idx[0] == 'h')
4219                         instr->type = INSTR_REGWR_RHI;
4220
4221                 instr->regarray.regarray_id = r->id;
4222                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4223                 instr->regarray.idx.n_bits = fidx->n_bits;
4224                 instr->regarray.idx.offset = fidx->offset / 8;
4225                 instr->regarray.dstsrc_val = src_val;
4226                 return 0;
4227         }
4228
4229         /* REGWR_RIH, REGWR_RIM. */
4230         if (!fidx && fsrc) {
4231                 idx_val = strtoul(idx, &idx, 0);
4232                 CHECK(!idx[0], EINVAL);
4233
4234                 CHECK(!fsrc->var_size, EINVAL);
4235
4236                 instr->type = INSTR_REGWR_RIM;
4237                 if (src[0] == 'h')
4238                         instr->type = INSTR_REGWR_RIH;
4239
4240                 instr->regarray.regarray_id = r->id;
4241                 instr->regarray.idx_val = idx_val;
4242                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4243                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4244                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4245                 return 0;
4246         }
4247
4248         /* REGWR_RII. */
4249         src_val = strtoull(src, &src, 0);
4250         CHECK(!src[0], EINVAL);
4251
4252         idx_val = strtoul(idx, &idx, 0);
4253         CHECK(!idx[0], EINVAL);
4254
4255         instr->type = INSTR_REGWR_RII;
4256         instr->regarray.idx_val = idx_val;
4257         instr->regarray.dstsrc_val = src_val;
4258
4259         return 0;
4260 }
4261
4262 static int
4263 instr_regadd_translate(struct rte_swx_pipeline *p,
4264                        struct action *action,
4265                        char **tokens,
4266                        int n_tokens,
4267                        struct instruction *instr,
4268                        struct instruction_data *data __rte_unused)
4269 {
4270         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4271         struct regarray *r;
4272         struct field *fidx, *fsrc;
4273         uint64_t src_val;
4274         uint32_t idx_struct_id, idx_val, src_struct_id;
4275
4276         CHECK(n_tokens == 4, EINVAL);
4277
4278         r = regarray_find(p, regarray);
4279         CHECK(r, EINVAL);
4280
4281         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4282         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4283         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4284         if (fidx && fsrc) {
4285                 CHECK(!fidx->var_size, EINVAL);
4286                 CHECK(!fsrc->var_size, EINVAL);
4287
4288                 instr->type = INSTR_REGADD_RMM;
4289                 if (idx[0] == 'h' && src[0] != 'h')
4290                         instr->type = INSTR_REGADD_RHM;
4291                 if (idx[0] != 'h' && src[0] == 'h')
4292                         instr->type = INSTR_REGADD_RMH;
4293                 if (idx[0] == 'h' && src[0] == 'h')
4294                         instr->type = INSTR_REGADD_RHH;
4295
4296                 instr->regarray.regarray_id = r->id;
4297                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4298                 instr->regarray.idx.n_bits = fidx->n_bits;
4299                 instr->regarray.idx.offset = fidx->offset / 8;
4300                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4301                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4302                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4303                 return 0;
4304         }
4305
4306         /* REGADD_RHI, REGADD_RMI. */
4307         if (fidx && !fsrc) {
4308                 CHECK(!fidx->var_size, EINVAL);
4309
4310                 src_val = strtoull(src, &src, 0);
4311                 CHECK(!src[0], EINVAL);
4312
4313                 instr->type = INSTR_REGADD_RMI;
4314                 if (idx[0] == 'h')
4315                         instr->type = INSTR_REGADD_RHI;
4316
4317                 instr->regarray.regarray_id = r->id;
4318                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4319                 instr->regarray.idx.n_bits = fidx->n_bits;
4320                 instr->regarray.idx.offset = fidx->offset / 8;
4321                 instr->regarray.dstsrc_val = src_val;
4322                 return 0;
4323         }
4324
4325         /* REGADD_RIH, REGADD_RIM. */
4326         if (!fidx && fsrc) {
4327                 idx_val = strtoul(idx, &idx, 0);
4328                 CHECK(!idx[0], EINVAL);
4329
4330                 CHECK(!fsrc->var_size, EINVAL);
4331
4332                 instr->type = INSTR_REGADD_RIM;
4333                 if (src[0] == 'h')
4334                         instr->type = INSTR_REGADD_RIH;
4335
4336                 instr->regarray.regarray_id = r->id;
4337                 instr->regarray.idx_val = idx_val;
4338                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4339                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4340                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4341                 return 0;
4342         }
4343
4344         /* REGADD_RII. */
4345         src_val = strtoull(src, &src, 0);
4346         CHECK(!src[0], EINVAL);
4347
4348         idx_val = strtoul(idx, &idx, 0);
4349         CHECK(!idx[0], EINVAL);
4350
4351         instr->type = INSTR_REGADD_RII;
4352         instr->regarray.idx_val = idx_val;
4353         instr->regarray.dstsrc_val = src_val;
4354         return 0;
4355 }
4356
4357 static inline void
4358 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4359 {
4360         struct thread *t = &p->threads[p->thread_id];
4361         struct instruction *ip = t->ip;
4362
4363         /* Structs. */
4364         __instr_regprefetch_rh_exec(p, t, ip);
4365
4366         /* Thread. */
4367         thread_ip_inc(p);
4368 }
4369
4370 static inline void
4371 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4372 {
4373         struct thread *t = &p->threads[p->thread_id];
4374         struct instruction *ip = t->ip;
4375
4376         /* Structs. */
4377         __instr_regprefetch_rm_exec(p, t, ip);
4378
4379         /* Thread. */
4380         thread_ip_inc(p);
4381 }
4382
4383 static inline void
4384 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4385 {
4386         struct thread *t = &p->threads[p->thread_id];
4387         struct instruction *ip = t->ip;
4388
4389         /* Structs. */
4390         __instr_regprefetch_ri_exec(p, t, ip);
4391
4392         /* Thread. */
4393         thread_ip_inc(p);
4394 }
4395
4396 static inline void
4397 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4398 {
4399         struct thread *t = &p->threads[p->thread_id];
4400         struct instruction *ip = t->ip;
4401
4402         /* Structs. */
4403         __instr_regrd_hrh_exec(p, t, ip);
4404
4405         /* Thread. */
4406         thread_ip_inc(p);
4407 }
4408
4409 static inline void
4410 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4411 {
4412         struct thread *t = &p->threads[p->thread_id];
4413         struct instruction *ip = t->ip;
4414
4415         /* Structs. */
4416         __instr_regrd_hrm_exec(p, t, ip);
4417
4418         /* Thread. */
4419         thread_ip_inc(p);
4420 }
4421
4422 static inline void
4423 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4424 {
4425         struct thread *t = &p->threads[p->thread_id];
4426         struct instruction *ip = t->ip;
4427
4428         /* Structs. */
4429         __instr_regrd_mrh_exec(p, t, ip);
4430
4431         /* Thread. */
4432         thread_ip_inc(p);
4433 }
4434
4435 static inline void
4436 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4437 {
4438         struct thread *t = &p->threads[p->thread_id];
4439         struct instruction *ip = t->ip;
4440
4441         /* Structs. */
4442         __instr_regrd_mrm_exec(p, t, ip);
4443
4444         /* Thread. */
4445         thread_ip_inc(p);
4446 }
4447
4448 static inline void
4449 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4450 {
4451         struct thread *t = &p->threads[p->thread_id];
4452         struct instruction *ip = t->ip;
4453
4454         /* Structs. */
4455         __instr_regrd_hri_exec(p, t, ip);
4456
4457         /* Thread. */
4458         thread_ip_inc(p);
4459 }
4460
4461 static inline void
4462 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4463 {
4464         struct thread *t = &p->threads[p->thread_id];
4465         struct instruction *ip = t->ip;
4466
4467         /* Structs. */
4468         __instr_regrd_mri_exec(p, t, ip);
4469
4470         /* Thread. */
4471         thread_ip_inc(p);
4472 }
4473
4474 static inline void
4475 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4476 {
4477         struct thread *t = &p->threads[p->thread_id];
4478         struct instruction *ip = t->ip;
4479
4480         /* Structs. */
4481         __instr_regwr_rhh_exec(p, t, ip);
4482
4483         /* Thread. */
4484         thread_ip_inc(p);
4485 }
4486
4487 static inline void
4488 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4489 {
4490         struct thread *t = &p->threads[p->thread_id];
4491         struct instruction *ip = t->ip;
4492
4493         /* Structs. */
4494         __instr_regwr_rhm_exec(p, t, ip);
4495
4496         /* Thread. */
4497         thread_ip_inc(p);
4498 }
4499
4500 static inline void
4501 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4502 {
4503         struct thread *t = &p->threads[p->thread_id];
4504         struct instruction *ip = t->ip;
4505
4506         /* Structs. */
4507         __instr_regwr_rmh_exec(p, t, ip);
4508
4509         /* Thread. */
4510         thread_ip_inc(p);
4511 }
4512
4513 static inline void
4514 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4515 {
4516         struct thread *t = &p->threads[p->thread_id];
4517         struct instruction *ip = t->ip;
4518
4519         /* Structs. */
4520         __instr_regwr_rmm_exec(p, t, ip);
4521
4522         /* Thread. */
4523         thread_ip_inc(p);
4524 }
4525
4526 static inline void
4527 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4528 {
4529         struct thread *t = &p->threads[p->thread_id];
4530         struct instruction *ip = t->ip;
4531
4532         /* Structs. */
4533         __instr_regwr_rhi_exec(p, t, ip);
4534
4535         /* Thread. */
4536         thread_ip_inc(p);
4537 }
4538
4539 static inline void
4540 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4541 {
4542         struct thread *t = &p->threads[p->thread_id];
4543         struct instruction *ip = t->ip;
4544
4545         /* Structs. */
4546         __instr_regwr_rmi_exec(p, t, ip);
4547
4548         /* Thread. */
4549         thread_ip_inc(p);
4550 }
4551
4552 static inline void
4553 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4554 {
4555         struct thread *t = &p->threads[p->thread_id];
4556         struct instruction *ip = t->ip;
4557
4558         /* Structs. */
4559         __instr_regwr_rih_exec(p, t, ip);
4560
4561         /* Thread. */
4562         thread_ip_inc(p);
4563 }
4564
4565 static inline void
4566 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4567 {
4568         struct thread *t = &p->threads[p->thread_id];
4569         struct instruction *ip = t->ip;
4570
4571         /* Structs. */
4572         __instr_regwr_rim_exec(p, t, ip);
4573
4574         /* Thread. */
4575         thread_ip_inc(p);
4576 }
4577
4578 static inline void
4579 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4580 {
4581         struct thread *t = &p->threads[p->thread_id];
4582         struct instruction *ip = t->ip;
4583
4584         /* Structs. */
4585         __instr_regwr_rii_exec(p, t, ip);
4586
4587         /* Thread. */
4588         thread_ip_inc(p);
4589 }
4590
4591 static inline void
4592 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4593 {
4594         struct thread *t = &p->threads[p->thread_id];
4595         struct instruction *ip = t->ip;
4596
4597         /* Structs. */
4598         __instr_regadd_rhh_exec(p, t, ip);
4599
4600         /* Thread. */
4601         thread_ip_inc(p);
4602 }
4603
4604 static inline void
4605 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4606 {
4607         struct thread *t = &p->threads[p->thread_id];
4608         struct instruction *ip = t->ip;
4609
4610         /* Structs. */
4611         __instr_regadd_rhm_exec(p, t, ip);
4612
4613         /* Thread. */
4614         thread_ip_inc(p);
4615 }
4616
4617 static inline void
4618 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4619 {
4620         struct thread *t = &p->threads[p->thread_id];
4621         struct instruction *ip = t->ip;
4622
4623         /* Structs. */
4624         __instr_regadd_rmh_exec(p, t, ip);
4625
4626         /* Thread. */
4627         thread_ip_inc(p);
4628 }
4629
4630 static inline void
4631 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4632 {
4633         struct thread *t = &p->threads[p->thread_id];
4634         struct instruction *ip = t->ip;
4635
4636         /* Structs. */
4637         __instr_regadd_rmm_exec(p, t, ip);
4638
4639         /* Thread. */
4640         thread_ip_inc(p);
4641 }
4642
4643 static inline void
4644 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4645 {
4646         struct thread *t = &p->threads[p->thread_id];
4647         struct instruction *ip = t->ip;
4648
4649         /* Structs. */
4650         __instr_regadd_rhi_exec(p, t, ip);
4651
4652         /* Thread. */
4653         thread_ip_inc(p);
4654 }
4655
4656 static inline void
4657 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4658 {
4659         struct thread *t = &p->threads[p->thread_id];
4660         struct instruction *ip = t->ip;
4661
4662         /* Structs. */
4663         __instr_regadd_rmi_exec(p, t, ip);
4664
4665         /* Thread. */
4666         thread_ip_inc(p);
4667 }
4668
4669 static inline void
4670 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4671 {
4672         struct thread *t = &p->threads[p->thread_id];
4673         struct instruction *ip = t->ip;
4674
4675         /* Structs. */
4676         __instr_regadd_rih_exec(p, t, ip);
4677
4678         /* Thread. */
4679         thread_ip_inc(p);
4680 }
4681
4682 static inline void
4683 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4684 {
4685         struct thread *t = &p->threads[p->thread_id];
4686         struct instruction *ip = t->ip;
4687
4688         /* Structs. */
4689         __instr_regadd_rim_exec(p, t, ip);
4690
4691         /* Thread. */
4692         thread_ip_inc(p);
4693 }
4694
4695 static inline void
4696 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4697 {
4698         struct thread *t = &p->threads[p->thread_id];
4699         struct instruction *ip = t->ip;
4700
4701         /* Structs. */
4702         __instr_regadd_rii_exec(p, t, ip);
4703
4704         /* Thread. */
4705         thread_ip_inc(p);
4706 }
4707
4708 /*
4709  * metarray.
4710  */
4711 static struct metarray *
4712 metarray_find(struct rte_swx_pipeline *p, const char *name);
4713
4714 static int
4715 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4716                             struct action *action,
4717                             char **tokens,
4718                             int n_tokens,
4719                             struct instruction *instr,
4720                             struct instruction_data *data __rte_unused)
4721 {
4722         char *metarray = tokens[1], *idx = tokens[2];
4723         struct metarray *m;
4724         struct field *fidx;
4725         uint32_t idx_struct_id, idx_val;
4726
4727         CHECK(n_tokens == 3, EINVAL);
4728
4729         m = metarray_find(p, metarray);
4730         CHECK(m, EINVAL);
4731
4732         /* METPREFETCH_H, METPREFETCH_M. */
4733         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4734         if (fidx) {
4735                 CHECK(!fidx->var_size, EINVAL);
4736
4737                 instr->type = INSTR_METPREFETCH_M;
4738                 if (idx[0] == 'h')
4739                         instr->type = INSTR_METPREFETCH_H;
4740
4741                 instr->meter.metarray_id = m->id;
4742                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4743                 instr->meter.idx.n_bits = fidx->n_bits;
4744                 instr->meter.idx.offset = fidx->offset / 8;
4745                 return 0;
4746         }
4747
4748         /* METPREFETCH_I. */
4749         idx_val = strtoul(idx, &idx, 0);
4750         CHECK(!idx[0], EINVAL);
4751
4752         instr->type = INSTR_METPREFETCH_I;
4753         instr->meter.metarray_id = m->id;
4754         instr->meter.idx_val = idx_val;
4755         return 0;
4756 }
4757
4758 static int
4759 instr_meter_translate(struct rte_swx_pipeline *p,
4760                       struct action *action,
4761                       char **tokens,
4762                       int n_tokens,
4763                       struct instruction *instr,
4764                       struct instruction_data *data __rte_unused)
4765 {
4766         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4767         char *color_in = tokens[4], *color_out = tokens[5];
4768         struct metarray *m;
4769         struct field *fidx, *flength, *fcin, *fcout;
4770         uint32_t idx_struct_id, length_struct_id;
4771         uint32_t color_in_struct_id, color_out_struct_id;
4772
4773         CHECK(n_tokens == 6, EINVAL);
4774
4775         m = metarray_find(p, metarray);
4776         CHECK(m, EINVAL);
4777
4778         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4779
4780         flength = struct_field_parse(p, action, length, &length_struct_id);
4781         CHECK(flength, EINVAL);
4782         CHECK(!flength->var_size, EINVAL);
4783
4784         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4785
4786         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4787         CHECK(fcout, EINVAL);
4788         CHECK(!fcout->var_size, EINVAL);
4789
4790         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4791         if (fidx && fcin) {
4792                 CHECK(!fidx->var_size, EINVAL);
4793                 CHECK(!fcin->var_size, EINVAL);
4794
4795                 instr->type = INSTR_METER_MMM;
4796                 if (idx[0] == 'h' && length[0] == 'h')
4797                         instr->type = INSTR_METER_HHM;
4798                 if (idx[0] == 'h' && length[0] != 'h')
4799                         instr->type = INSTR_METER_HMM;
4800                 if (idx[0] != 'h' && length[0] == 'h')
4801                         instr->type = INSTR_METER_MHM;
4802
4803                 instr->meter.metarray_id = m->id;
4804
4805                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4806                 instr->meter.idx.n_bits = fidx->n_bits;
4807                 instr->meter.idx.offset = fidx->offset / 8;
4808
4809                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4810                 instr->meter.length.n_bits = flength->n_bits;
4811                 instr->meter.length.offset = flength->offset / 8;
4812
4813                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4814                 instr->meter.color_in.n_bits = fcin->n_bits;
4815                 instr->meter.color_in.offset = fcin->offset / 8;
4816
4817                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4818                 instr->meter.color_out.n_bits = fcout->n_bits;
4819                 instr->meter.color_out.offset = fcout->offset / 8;
4820         }
4821
4822         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4823         if (fidx && !fcin) {
4824                 uint32_t color_in_val;
4825
4826                 CHECK(!fidx->var_size, EINVAL);
4827
4828                 color_in_val = strtoul(color_in, &color_in, 0);
4829                 CHECK(!color_in[0], EINVAL);
4830
4831                 instr->type = INSTR_METER_MMI;
4832                 if (idx[0] == 'h' && length[0] == 'h')
4833                         instr->type = INSTR_METER_HHI;
4834                 if (idx[0] == 'h' && length[0] != 'h')
4835                         instr->type = INSTR_METER_HMI;
4836                 if (idx[0] != 'h' && length[0] == 'h')
4837                         instr->type = INSTR_METER_MHI;
4838
4839                 instr->meter.metarray_id = m->id;
4840
4841                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4842                 instr->meter.idx.n_bits = fidx->n_bits;
4843                 instr->meter.idx.offset = fidx->offset / 8;
4844
4845                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4846                 instr->meter.length.n_bits = flength->n_bits;
4847                 instr->meter.length.offset = flength->offset / 8;
4848
4849                 instr->meter.color_in_val = color_in_val;
4850
4851                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4852                 instr->meter.color_out.n_bits = fcout->n_bits;
4853                 instr->meter.color_out.offset = fcout->offset / 8;
4854         }
4855
4856         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4857         if (!fidx && fcin) {
4858                 uint32_t idx_val;
4859
4860                 idx_val = strtoul(idx, &idx, 0);
4861                 CHECK(!idx[0], EINVAL);
4862
4863                 CHECK(!fcin->var_size, EINVAL);
4864
4865                 instr->type = INSTR_METER_IMM;
4866                 if (length[0] == 'h')
4867                         instr->type = INSTR_METER_IHM;
4868
4869                 instr->meter.metarray_id = m->id;
4870
4871                 instr->meter.idx_val = idx_val;
4872
4873                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4874                 instr->meter.length.n_bits = flength->n_bits;
4875                 instr->meter.length.offset = flength->offset / 8;
4876
4877                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4878                 instr->meter.color_in.n_bits = fcin->n_bits;
4879                 instr->meter.color_in.offset = fcin->offset / 8;
4880
4881                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4882                 instr->meter.color_out.n_bits = fcout->n_bits;
4883                 instr->meter.color_out.offset = fcout->offset / 8;
4884         }
4885
4886         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4887         if (!fidx && !fcin) {
4888                 uint32_t idx_val, color_in_val;
4889
4890                 idx_val = strtoul(idx, &idx, 0);
4891                 CHECK(!idx[0], EINVAL);
4892
4893                 color_in_val = strtoul(color_in, &color_in, 0);
4894                 CHECK(!color_in[0], EINVAL);
4895
4896                 instr->type = INSTR_METER_IMI;
4897                 if (length[0] == 'h')
4898                         instr->type = INSTR_METER_IHI;
4899
4900                 instr->meter.metarray_id = m->id;
4901
4902                 instr->meter.idx_val = idx_val;
4903
4904                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4905                 instr->meter.length.n_bits = flength->n_bits;
4906                 instr->meter.length.offset = flength->offset / 8;
4907
4908                 instr->meter.color_in_val = color_in_val;
4909
4910                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4911                 instr->meter.color_out.n_bits = fcout->n_bits;
4912                 instr->meter.color_out.offset = fcout->offset / 8;
4913         }
4914
4915         return 0;
4916 }
4917
4918 static inline void
4919 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4920 {
4921         struct thread *t = &p->threads[p->thread_id];
4922         struct instruction *ip = t->ip;
4923
4924         /* Structs. */
4925         __instr_metprefetch_h_exec(p, t, ip);
4926
4927         /* Thread. */
4928         thread_ip_inc(p);
4929 }
4930
4931 static inline void
4932 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4933 {
4934         struct thread *t = &p->threads[p->thread_id];
4935         struct instruction *ip = t->ip;
4936
4937         /* Structs. */
4938         __instr_metprefetch_m_exec(p, t, ip);
4939
4940         /* Thread. */
4941         thread_ip_inc(p);
4942 }
4943
4944 static inline void
4945 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4946 {
4947         struct thread *t = &p->threads[p->thread_id];
4948         struct instruction *ip = t->ip;
4949
4950         /* Structs. */
4951         __instr_metprefetch_i_exec(p, t, ip);
4952
4953         /* Thread. */
4954         thread_ip_inc(p);
4955 }
4956
4957 static inline void
4958 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4959 {
4960         struct thread *t = &p->threads[p->thread_id];
4961         struct instruction *ip = t->ip;
4962
4963         /* Structs. */
4964         __instr_meter_hhm_exec(p, t, ip);
4965
4966         /* Thread. */
4967         thread_ip_inc(p);
4968 }
4969
4970 static inline void
4971 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4972 {
4973         struct thread *t = &p->threads[p->thread_id];
4974         struct instruction *ip = t->ip;
4975
4976         /* Structs. */
4977         __instr_meter_hhi_exec(p, t, ip);
4978
4979         /* Thread. */
4980         thread_ip_inc(p);
4981 }
4982
4983 static inline void
4984 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4985 {
4986         struct thread *t = &p->threads[p->thread_id];
4987         struct instruction *ip = t->ip;
4988
4989         /* Structs. */
4990         __instr_meter_hmm_exec(p, t, ip);
4991
4992         /* Thread. */
4993         thread_ip_inc(p);
4994 }
4995
4996 static inline void
4997 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4998 {
4999         struct thread *t = &p->threads[p->thread_id];
5000         struct instruction *ip = t->ip;
5001
5002         /* Structs. */
5003         __instr_meter_hmi_exec(p, t, ip);
5004
5005         /* Thread. */
5006         thread_ip_inc(p);
5007 }
5008
5009 static inline void
5010 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5011 {
5012         struct thread *t = &p->threads[p->thread_id];
5013         struct instruction *ip = t->ip;
5014
5015         /* Structs. */
5016         __instr_meter_mhm_exec(p, t, ip);
5017
5018         /* Thread. */
5019         thread_ip_inc(p);
5020 }
5021
5022 static inline void
5023 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5024 {
5025         struct thread *t = &p->threads[p->thread_id];
5026         struct instruction *ip = t->ip;
5027
5028         /* Structs. */
5029         __instr_meter_mhi_exec(p, t, ip);
5030
5031         /* Thread. */
5032         thread_ip_inc(p);
5033 }
5034
5035 static inline void
5036 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5037 {
5038         struct thread *t = &p->threads[p->thread_id];
5039         struct instruction *ip = t->ip;
5040
5041         /* Structs. */
5042         __instr_meter_mmm_exec(p, t, ip);
5043
5044         /* Thread. */
5045         thread_ip_inc(p);
5046 }
5047
5048 static inline void
5049 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5050 {
5051         struct thread *t = &p->threads[p->thread_id];
5052         struct instruction *ip = t->ip;
5053
5054         /* Structs. */
5055         __instr_meter_mmi_exec(p, t, ip);
5056
5057         /* Thread. */
5058         thread_ip_inc(p);
5059 }
5060
5061 static inline void
5062 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5063 {
5064         struct thread *t = &p->threads[p->thread_id];
5065         struct instruction *ip = t->ip;
5066
5067         /* Structs. */
5068         __instr_meter_ihm_exec(p, t, ip);
5069
5070         /* Thread. */
5071         thread_ip_inc(p);
5072 }
5073
5074 static inline void
5075 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5076 {
5077         struct thread *t = &p->threads[p->thread_id];
5078         struct instruction *ip = t->ip;
5079
5080         /* Structs. */
5081         __instr_meter_ihi_exec(p, t, ip);
5082
5083         /* Thread. */
5084         thread_ip_inc(p);
5085 }
5086
5087 static inline void
5088 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5089 {
5090         struct thread *t = &p->threads[p->thread_id];
5091         struct instruction *ip = t->ip;
5092
5093         /* Structs. */
5094         __instr_meter_imm_exec(p, t, ip);
5095
5096         /* Thread. */
5097         thread_ip_inc(p);
5098 }
5099
5100 static inline void
5101 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5102 {
5103         struct thread *t = &p->threads[p->thread_id];
5104         struct instruction *ip = t->ip;
5105
5106         /* Structs. */
5107         __instr_meter_imi_exec(p, t, ip);
5108
5109         /* Thread. */
5110         thread_ip_inc(p);
5111 }
5112
5113 /*
5114  * jmp.
5115  */
5116 static int
5117 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5118                     struct action *action __rte_unused,
5119                     char **tokens,
5120                     int n_tokens,
5121                     struct instruction *instr,
5122                     struct instruction_data *data)
5123 {
5124         CHECK(n_tokens == 2, EINVAL);
5125
5126         strcpy(data->jmp_label, tokens[1]);
5127
5128         instr->type = INSTR_JMP;
5129         instr->jmp.ip = NULL; /* Resolved later. */
5130         return 0;
5131 }
5132
5133 static int
5134 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5135                           struct action *action __rte_unused,
5136                           char **tokens,
5137                           int n_tokens,
5138                           struct instruction *instr,
5139                           struct instruction_data *data)
5140 {
5141         struct header *h;
5142
5143         CHECK(n_tokens == 3, EINVAL);
5144
5145         strcpy(data->jmp_label, tokens[1]);
5146
5147         h = header_parse(p, tokens[2]);
5148         CHECK(h, EINVAL);
5149
5150         instr->type = INSTR_JMP_VALID;
5151         instr->jmp.ip = NULL; /* Resolved later. */
5152         instr->jmp.header_id = h->id;
5153         return 0;
5154 }
5155
5156 static int
5157 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5158                             struct action *action __rte_unused,
5159                             char **tokens,
5160                             int n_tokens,
5161                             struct instruction *instr,
5162                             struct instruction_data *data)
5163 {
5164         struct header *h;
5165
5166         CHECK(n_tokens == 3, EINVAL);
5167
5168         strcpy(data->jmp_label, tokens[1]);
5169
5170         h = header_parse(p, tokens[2]);
5171         CHECK(h, EINVAL);
5172
5173         instr->type = INSTR_JMP_INVALID;
5174         instr->jmp.ip = NULL; /* Resolved later. */
5175         instr->jmp.header_id = h->id;
5176         return 0;
5177 }
5178
5179 static int
5180 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5181                         struct action *action,
5182                         char **tokens,
5183                         int n_tokens,
5184                         struct instruction *instr,
5185                         struct instruction_data *data)
5186 {
5187         CHECK(!action, EINVAL);
5188         CHECK(n_tokens == 2, EINVAL);
5189
5190         strcpy(data->jmp_label, tokens[1]);
5191
5192         instr->type = INSTR_JMP_HIT;
5193         instr->jmp.ip = NULL; /* Resolved later. */
5194         return 0;
5195 }
5196
5197 static int
5198 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5199                          struct action *action,
5200                          char **tokens,
5201                          int n_tokens,
5202                          struct instruction *instr,
5203                          struct instruction_data *data)
5204 {
5205         CHECK(!action, EINVAL);
5206         CHECK(n_tokens == 2, EINVAL);
5207
5208         strcpy(data->jmp_label, tokens[1]);
5209
5210         instr->type = INSTR_JMP_MISS;
5211         instr->jmp.ip = NULL; /* Resolved later. */
5212         return 0;
5213 }
5214
5215 static int
5216 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5217                                struct action *action,
5218                                char **tokens,
5219                                int n_tokens,
5220                                struct instruction *instr,
5221                                struct instruction_data *data)
5222 {
5223         struct action *a;
5224
5225         CHECK(!action, EINVAL);
5226         CHECK(n_tokens == 3, EINVAL);
5227
5228         strcpy(data->jmp_label, tokens[1]);
5229
5230         a = action_find(p, tokens[2]);
5231         CHECK(a, EINVAL);
5232
5233         instr->type = INSTR_JMP_ACTION_HIT;
5234         instr->jmp.ip = NULL; /* Resolved later. */
5235         instr->jmp.action_id = a->id;
5236         return 0;
5237 }
5238
5239 static int
5240 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5241                                 struct action *action,
5242                                 char **tokens,
5243                                 int n_tokens,
5244                                 struct instruction *instr,
5245                                 struct instruction_data *data)
5246 {
5247         struct action *a;
5248
5249         CHECK(!action, EINVAL);
5250         CHECK(n_tokens == 3, EINVAL);
5251
5252         strcpy(data->jmp_label, tokens[1]);
5253
5254         a = action_find(p, tokens[2]);
5255         CHECK(a, EINVAL);
5256
5257         instr->type = INSTR_JMP_ACTION_MISS;
5258         instr->jmp.ip = NULL; /* Resolved later. */
5259         instr->jmp.action_id = a->id;
5260         return 0;
5261 }
5262
5263 static int
5264 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5265                        struct action *action,
5266                        char **tokens,
5267                        int n_tokens,
5268                        struct instruction *instr,
5269                        struct instruction_data *data)
5270 {
5271         char *a = tokens[2], *b = tokens[3];
5272         struct field *fa, *fb;
5273         uint64_t b_val;
5274         uint32_t a_struct_id, b_struct_id;
5275
5276         CHECK(n_tokens == 4, EINVAL);
5277
5278         strcpy(data->jmp_label, tokens[1]);
5279
5280         fa = struct_field_parse(p, action, a, &a_struct_id);
5281         CHECK(fa, EINVAL);
5282         CHECK(!fa->var_size, EINVAL);
5283
5284         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5285         fb = struct_field_parse(p, action, b, &b_struct_id);
5286         if (fb) {
5287                 CHECK(!fb->var_size, EINVAL);
5288
5289                 instr->type = INSTR_JMP_EQ;
5290                 if (a[0] != 'h' && b[0] == 'h')
5291                         instr->type = INSTR_JMP_EQ_MH;
5292                 if (a[0] == 'h' && b[0] != 'h')
5293                         instr->type = INSTR_JMP_EQ_HM;
5294                 if (a[0] == 'h' && b[0] == 'h')
5295                         instr->type = INSTR_JMP_EQ_HH;
5296                 instr->jmp.ip = NULL; /* Resolved later. */
5297
5298                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5299                 instr->jmp.a.n_bits = fa->n_bits;
5300                 instr->jmp.a.offset = fa->offset / 8;
5301                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5302                 instr->jmp.b.n_bits = fb->n_bits;
5303                 instr->jmp.b.offset = fb->offset / 8;
5304                 return 0;
5305         }
5306
5307         /* JMP_EQ_I. */
5308         b_val = strtoull(b, &b, 0);
5309         CHECK(!b[0], EINVAL);
5310
5311         if (a[0] == 'h')
5312                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5313
5314         instr->type = INSTR_JMP_EQ_I;
5315         instr->jmp.ip = NULL; /* Resolved later. */
5316         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5317         instr->jmp.a.n_bits = fa->n_bits;
5318         instr->jmp.a.offset = fa->offset / 8;
5319         instr->jmp.b_val = b_val;
5320         return 0;
5321 }
5322
5323 static int
5324 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5325                         struct action *action,
5326                         char **tokens,
5327                         int n_tokens,
5328                         struct instruction *instr,
5329                         struct instruction_data *data)
5330 {
5331         char *a = tokens[2], *b = tokens[3];
5332         struct field *fa, *fb;
5333         uint64_t b_val;
5334         uint32_t a_struct_id, b_struct_id;
5335
5336         CHECK(n_tokens == 4, EINVAL);
5337
5338         strcpy(data->jmp_label, tokens[1]);
5339
5340         fa = struct_field_parse(p, action, a, &a_struct_id);
5341         CHECK(fa, EINVAL);
5342         CHECK(!fa->var_size, EINVAL);
5343
5344         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5345         fb = struct_field_parse(p, action, b, &b_struct_id);
5346         if (fb) {
5347                 CHECK(!fb->var_size, EINVAL);
5348
5349                 instr->type = INSTR_JMP_NEQ;
5350                 if (a[0] != 'h' && b[0] == 'h')
5351                         instr->type = INSTR_JMP_NEQ_MH;
5352                 if (a[0] == 'h' && b[0] != 'h')
5353                         instr->type = INSTR_JMP_NEQ_HM;
5354                 if (a[0] == 'h' && b[0] == 'h')
5355                         instr->type = INSTR_JMP_NEQ_HH;
5356                 instr->jmp.ip = NULL; /* Resolved later. */
5357
5358                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5359                 instr->jmp.a.n_bits = fa->n_bits;
5360                 instr->jmp.a.offset = fa->offset / 8;
5361                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5362                 instr->jmp.b.n_bits = fb->n_bits;
5363                 instr->jmp.b.offset = fb->offset / 8;
5364                 return 0;
5365         }
5366
5367         /* JMP_NEQ_I. */
5368         b_val = strtoull(b, &b, 0);
5369         CHECK(!b[0], EINVAL);
5370
5371         if (a[0] == 'h')
5372                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5373
5374         instr->type = INSTR_JMP_NEQ_I;
5375         instr->jmp.ip = NULL; /* Resolved later. */
5376         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5377         instr->jmp.a.n_bits = fa->n_bits;
5378         instr->jmp.a.offset = fa->offset / 8;
5379         instr->jmp.b_val = b_val;
5380         return 0;
5381 }
5382
5383 static int
5384 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5385                        struct action *action,
5386                        char **tokens,
5387                        int n_tokens,
5388                        struct instruction *instr,
5389                        struct instruction_data *data)
5390 {
5391         char *a = tokens[2], *b = tokens[3];
5392         struct field *fa, *fb;
5393         uint64_t b_val;
5394         uint32_t a_struct_id, b_struct_id;
5395
5396         CHECK(n_tokens == 4, EINVAL);
5397
5398         strcpy(data->jmp_label, tokens[1]);
5399
5400         fa = struct_field_parse(p, action, a, &a_struct_id);
5401         CHECK(fa, EINVAL);
5402         CHECK(!fa->var_size, EINVAL);
5403
5404         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5405         fb = struct_field_parse(p, action, b, &b_struct_id);
5406         if (fb) {
5407                 CHECK(!fb->var_size, EINVAL);
5408
5409                 instr->type = INSTR_JMP_LT;
5410                 if (a[0] == 'h' && b[0] != 'h')
5411                         instr->type = INSTR_JMP_LT_HM;
5412                 if (a[0] != 'h' && b[0] == 'h')
5413                         instr->type = INSTR_JMP_LT_MH;
5414                 if (a[0] == 'h' && b[0] == 'h')
5415                         instr->type = INSTR_JMP_LT_HH;
5416                 instr->jmp.ip = NULL; /* Resolved later. */
5417
5418                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5419                 instr->jmp.a.n_bits = fa->n_bits;
5420                 instr->jmp.a.offset = fa->offset / 8;
5421                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5422                 instr->jmp.b.n_bits = fb->n_bits;
5423                 instr->jmp.b.offset = fb->offset / 8;
5424                 return 0;
5425         }
5426
5427         /* JMP_LT_MI, JMP_LT_HI. */
5428         b_val = strtoull(b, &b, 0);
5429         CHECK(!b[0], EINVAL);
5430
5431         instr->type = INSTR_JMP_LT_MI;
5432         if (a[0] == 'h')
5433                 instr->type = INSTR_JMP_LT_HI;
5434         instr->jmp.ip = NULL; /* Resolved later. */
5435
5436         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5437         instr->jmp.a.n_bits = fa->n_bits;
5438         instr->jmp.a.offset = fa->offset / 8;
5439         instr->jmp.b_val = b_val;
5440         return 0;
5441 }
5442
5443 static int
5444 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5445                        struct action *action,
5446                        char **tokens,
5447                        int n_tokens,
5448                        struct instruction *instr,
5449                        struct instruction_data *data)
5450 {
5451         char *a = tokens[2], *b = tokens[3];
5452         struct field *fa, *fb;
5453         uint64_t b_val;
5454         uint32_t a_struct_id, b_struct_id;
5455
5456         CHECK(n_tokens == 4, EINVAL);
5457
5458         strcpy(data->jmp_label, tokens[1]);
5459
5460         fa = struct_field_parse(p, action, a, &a_struct_id);
5461         CHECK(fa, EINVAL);
5462         CHECK(!fa->var_size, EINVAL);
5463
5464         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5465         fb = struct_field_parse(p, action, b, &b_struct_id);
5466         if (fb) {
5467                 CHECK(!fb->var_size, EINVAL);
5468
5469                 instr->type = INSTR_JMP_GT;
5470                 if (a[0] == 'h' && b[0] != 'h')
5471                         instr->type = INSTR_JMP_GT_HM;
5472                 if (a[0] != 'h' && b[0] == 'h')
5473                         instr->type = INSTR_JMP_GT_MH;
5474                 if (a[0] == 'h' && b[0] == 'h')
5475                         instr->type = INSTR_JMP_GT_HH;
5476                 instr->jmp.ip = NULL; /* Resolved later. */
5477
5478                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5479                 instr->jmp.a.n_bits = fa->n_bits;
5480                 instr->jmp.a.offset = fa->offset / 8;
5481                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5482                 instr->jmp.b.n_bits = fb->n_bits;
5483                 instr->jmp.b.offset = fb->offset / 8;
5484                 return 0;
5485         }
5486
5487         /* JMP_GT_MI, JMP_GT_HI. */
5488         b_val = strtoull(b, &b, 0);
5489         CHECK(!b[0], EINVAL);
5490
5491         instr->type = INSTR_JMP_GT_MI;
5492         if (a[0] == 'h')
5493                 instr->type = INSTR_JMP_GT_HI;
5494         instr->jmp.ip = NULL; /* Resolved later. */
5495
5496         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5497         instr->jmp.a.n_bits = fa->n_bits;
5498         instr->jmp.a.offset = fa->offset / 8;
5499         instr->jmp.b_val = b_val;
5500         return 0;
5501 }
5502
5503 static inline void
5504 instr_jmp_exec(struct rte_swx_pipeline *p)
5505 {
5506         struct thread *t = &p->threads[p->thread_id];
5507         struct instruction *ip = t->ip;
5508
5509         TRACE("[Thread %2u] jmp\n", p->thread_id);
5510
5511         thread_ip_set(t, ip->jmp.ip);
5512 }
5513
5514 static inline void
5515 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5516 {
5517         struct thread *t = &p->threads[p->thread_id];
5518         struct instruction *ip = t->ip;
5519         uint32_t header_id = ip->jmp.header_id;
5520
5521         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5522
5523         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5524 }
5525
5526 static inline void
5527 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5528 {
5529         struct thread *t = &p->threads[p->thread_id];
5530         struct instruction *ip = t->ip;
5531         uint32_t header_id = ip->jmp.header_id;
5532
5533         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5534
5535         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5536 }
5537
5538 static inline void
5539 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5540 {
5541         struct thread *t = &p->threads[p->thread_id];
5542         struct instruction *ip = t->ip;
5543         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5544
5545         TRACE("[Thread %2u] jmph\n", p->thread_id);
5546
5547         t->ip = ip_next[t->hit];
5548 }
5549
5550 static inline void
5551 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5552 {
5553         struct thread *t = &p->threads[p->thread_id];
5554         struct instruction *ip = t->ip;
5555         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5556
5557         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5558
5559         t->ip = ip_next[t->hit];
5560 }
5561
5562 static inline void
5563 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5564 {
5565         struct thread *t = &p->threads[p->thread_id];
5566         struct instruction *ip = t->ip;
5567
5568         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5569
5570         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5571 }
5572
5573 static inline void
5574 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5575 {
5576         struct thread *t = &p->threads[p->thread_id];
5577         struct instruction *ip = t->ip;
5578
5579         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5580
5581         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5582 }
5583
5584 static inline void
5585 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5586 {
5587         struct thread *t = &p->threads[p->thread_id];
5588         struct instruction *ip = t->ip;
5589
5590         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5591
5592         JMP_CMP(t, ip, ==);
5593 }
5594
5595 static inline void
5596 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5597 {
5598         struct thread *t = &p->threads[p->thread_id];
5599         struct instruction *ip = t->ip;
5600
5601         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5602
5603         JMP_CMP_MH(t, ip, ==);
5604 }
5605
5606 static inline void
5607 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5608 {
5609         struct thread *t = &p->threads[p->thread_id];
5610         struct instruction *ip = t->ip;
5611
5612         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5613
5614         JMP_CMP_HM(t, ip, ==);
5615 }
5616
5617 static inline void
5618 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5619 {
5620         struct thread *t = &p->threads[p->thread_id];
5621         struct instruction *ip = t->ip;
5622
5623         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5624
5625         JMP_CMP_HH_FAST(t, ip, ==);
5626 }
5627
5628 static inline void
5629 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5630 {
5631         struct thread *t = &p->threads[p->thread_id];
5632         struct instruction *ip = t->ip;
5633
5634         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5635
5636         JMP_CMP_I(t, ip, ==);
5637 }
5638
5639 static inline void
5640 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5641 {
5642         struct thread *t = &p->threads[p->thread_id];
5643         struct instruction *ip = t->ip;
5644
5645         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5646
5647         JMP_CMP(t, ip, !=);
5648 }
5649
5650 static inline void
5651 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5652 {
5653         struct thread *t = &p->threads[p->thread_id];
5654         struct instruction *ip = t->ip;
5655
5656         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5657
5658         JMP_CMP_MH(t, ip, !=);
5659 }
5660
5661 static inline void
5662 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5663 {
5664         struct thread *t = &p->threads[p->thread_id];
5665         struct instruction *ip = t->ip;
5666
5667         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5668
5669         JMP_CMP_HM(t, ip, !=);
5670 }
5671
5672 static inline void
5673 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5674 {
5675         struct thread *t = &p->threads[p->thread_id];
5676         struct instruction *ip = t->ip;
5677
5678         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5679
5680         JMP_CMP_HH_FAST(t, ip, !=);
5681 }
5682
5683 static inline void
5684 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5685 {
5686         struct thread *t = &p->threads[p->thread_id];
5687         struct instruction *ip = t->ip;
5688
5689         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5690
5691         JMP_CMP_I(t, ip, !=);
5692 }
5693
5694 static inline void
5695 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5696 {
5697         struct thread *t = &p->threads[p->thread_id];
5698         struct instruction *ip = t->ip;
5699
5700         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5701
5702         JMP_CMP(t, ip, <);
5703 }
5704
5705 static inline void
5706 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5707 {
5708         struct thread *t = &p->threads[p->thread_id];
5709         struct instruction *ip = t->ip;
5710
5711         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5712
5713         JMP_CMP_MH(t, ip, <);
5714 }
5715
5716 static inline void
5717 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5718 {
5719         struct thread *t = &p->threads[p->thread_id];
5720         struct instruction *ip = t->ip;
5721
5722         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5723
5724         JMP_CMP_HM(t, ip, <);
5725 }
5726
5727 static inline void
5728 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5729 {
5730         struct thread *t = &p->threads[p->thread_id];
5731         struct instruction *ip = t->ip;
5732
5733         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5734
5735         JMP_CMP_HH(t, ip, <);
5736 }
5737
5738 static inline void
5739 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5740 {
5741         struct thread *t = &p->threads[p->thread_id];
5742         struct instruction *ip = t->ip;
5743
5744         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5745
5746         JMP_CMP_MI(t, ip, <);
5747 }
5748
5749 static inline void
5750 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5751 {
5752         struct thread *t = &p->threads[p->thread_id];
5753         struct instruction *ip = t->ip;
5754
5755         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5756
5757         JMP_CMP_HI(t, ip, <);
5758 }
5759
5760 static inline void
5761 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5762 {
5763         struct thread *t = &p->threads[p->thread_id];
5764         struct instruction *ip = t->ip;
5765
5766         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5767
5768         JMP_CMP(t, ip, >);
5769 }
5770
5771 static inline void
5772 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5773 {
5774         struct thread *t = &p->threads[p->thread_id];
5775         struct instruction *ip = t->ip;
5776
5777         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5778
5779         JMP_CMP_MH(t, ip, >);
5780 }
5781
5782 static inline void
5783 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5784 {
5785         struct thread *t = &p->threads[p->thread_id];
5786         struct instruction *ip = t->ip;
5787
5788         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5789
5790         JMP_CMP_HM(t, ip, >);
5791 }
5792
5793 static inline void
5794 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5795 {
5796         struct thread *t = &p->threads[p->thread_id];
5797         struct instruction *ip = t->ip;
5798
5799         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5800
5801         JMP_CMP_HH(t, ip, >);
5802 }
5803
5804 static inline void
5805 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5806 {
5807         struct thread *t = &p->threads[p->thread_id];
5808         struct instruction *ip = t->ip;
5809
5810         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5811
5812         JMP_CMP_MI(t, ip, >);
5813 }
5814
5815 static inline void
5816 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5817 {
5818         struct thread *t = &p->threads[p->thread_id];
5819         struct instruction *ip = t->ip;
5820
5821         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5822
5823         JMP_CMP_HI(t, ip, >);
5824 }
5825
5826 /*
5827  * return.
5828  */
5829 static int
5830 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5831                        struct action *action,
5832                        char **tokens __rte_unused,
5833                        int n_tokens,
5834                        struct instruction *instr,
5835                        struct instruction_data *data __rte_unused)
5836 {
5837         CHECK(action, EINVAL);
5838         CHECK(n_tokens == 1, EINVAL);
5839
5840         instr->type = INSTR_RETURN;
5841         return 0;
5842 }
5843
5844 static inline void
5845 instr_return_exec(struct rte_swx_pipeline *p)
5846 {
5847         struct thread *t = &p->threads[p->thread_id];
5848
5849         TRACE("[Thread %2u] return\n", p->thread_id);
5850
5851         t->ip = t->ret;
5852 }
5853
5854 static int
5855 instr_translate(struct rte_swx_pipeline *p,
5856                 struct action *action,
5857                 char *string,
5858                 struct instruction *instr,
5859                 struct instruction_data *data)
5860 {
5861         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5862         int n_tokens = 0, tpos = 0;
5863
5864         /* Parse the instruction string into tokens. */
5865         for ( ; ; ) {
5866                 char *token;
5867
5868                 token = strtok_r(string, " \t\v", &string);
5869                 if (!token)
5870                         break;
5871
5872                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5873                 CHECK_NAME(token, EINVAL);
5874
5875                 tokens[n_tokens] = token;
5876                 n_tokens++;
5877         }
5878
5879         CHECK(n_tokens, EINVAL);
5880
5881         /* Handle the optional instruction label. */
5882         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5883                 strcpy(data->label, tokens[0]);
5884
5885                 tpos += 2;
5886                 CHECK(n_tokens - tpos, EINVAL);
5887         }
5888
5889         /* Identify the instruction type. */
5890         if (!strcmp(tokens[tpos], "rx"))
5891                 return instr_rx_translate(p,
5892                                           action,
5893                                           &tokens[tpos],
5894                                           n_tokens - tpos,
5895                                           instr,
5896                                           data);
5897
5898         if (!strcmp(tokens[tpos], "tx"))
5899                 return instr_tx_translate(p,
5900                                           action,
5901                                           &tokens[tpos],
5902                                           n_tokens - tpos,
5903                                           instr,
5904                                           data);
5905
5906         if (!strcmp(tokens[tpos], "drop"))
5907                 return instr_drop_translate(p,
5908                                             action,
5909                                             &tokens[tpos],
5910                                             n_tokens - tpos,
5911                                             instr,
5912                                             data);
5913
5914         if (!strcmp(tokens[tpos], "mirror"))
5915                 return instr_mirror_translate(p,
5916                                               action,
5917                                               &tokens[tpos],
5918                                               n_tokens - tpos,
5919                                               instr,
5920                                               data);
5921
5922         if (!strcmp(tokens[tpos], "recirculate"))
5923                 return instr_recirculate_translate(p,
5924                                               action,
5925                                               &tokens[tpos],
5926                                               n_tokens - tpos,
5927                                               instr,
5928                                               data);
5929
5930         if (!strcmp(tokens[tpos], "recircid"))
5931                 return instr_recircid_translate(p,
5932                                               action,
5933                                               &tokens[tpos],
5934                                               n_tokens - tpos,
5935                                               instr,
5936                                               data);
5937
5938         if (!strcmp(tokens[tpos], "extract"))
5939                 return instr_hdr_extract_translate(p,
5940                                                    action,
5941                                                    &tokens[tpos],
5942                                                    n_tokens - tpos,
5943                                                    instr,
5944                                                    data);
5945
5946         if (!strcmp(tokens[tpos], "lookahead"))
5947                 return instr_hdr_lookahead_translate(p,
5948                                                      action,
5949                                                      &tokens[tpos],
5950                                                      n_tokens - tpos,
5951                                                      instr,
5952                                                      data);
5953
5954         if (!strcmp(tokens[tpos], "emit"))
5955                 return instr_hdr_emit_translate(p,
5956                                                 action,
5957                                                 &tokens[tpos],
5958                                                 n_tokens - tpos,
5959                                                 instr,
5960                                                 data);
5961
5962         if (!strcmp(tokens[tpos], "validate"))
5963                 return instr_hdr_validate_translate(p,
5964                                                     action,
5965                                                     &tokens[tpos],
5966                                                     n_tokens - tpos,
5967                                                     instr,
5968                                                     data);
5969
5970         if (!strcmp(tokens[tpos], "invalidate"))
5971                 return instr_hdr_invalidate_translate(p,
5972                                                       action,
5973                                                       &tokens[tpos],
5974                                                       n_tokens - tpos,
5975                                                       instr,
5976                                                       data);
5977
5978         if (!strcmp(tokens[tpos], "mov"))
5979                 return instr_mov_translate(p,
5980                                            action,
5981                                            &tokens[tpos],
5982                                            n_tokens - tpos,
5983                                            instr,
5984                                            data);
5985
5986         if (!strcmp(tokens[tpos], "add"))
5987                 return instr_alu_add_translate(p,
5988                                                action,
5989                                                &tokens[tpos],
5990                                                n_tokens - tpos,
5991                                                instr,
5992                                                data);
5993
5994         if (!strcmp(tokens[tpos], "sub"))
5995                 return instr_alu_sub_translate(p,
5996                                                action,
5997                                                &tokens[tpos],
5998                                                n_tokens - tpos,
5999                                                instr,
6000                                                data);
6001
6002         if (!strcmp(tokens[tpos], "ckadd"))
6003                 return instr_alu_ckadd_translate(p,
6004                                                  action,
6005                                                  &tokens[tpos],
6006                                                  n_tokens - tpos,
6007                                                  instr,
6008                                                  data);
6009
6010         if (!strcmp(tokens[tpos], "cksub"))
6011                 return instr_alu_cksub_translate(p,
6012                                                  action,
6013                                                  &tokens[tpos],
6014                                                  n_tokens - tpos,
6015                                                  instr,
6016                                                  data);
6017
6018         if (!strcmp(tokens[tpos], "and"))
6019                 return instr_alu_and_translate(p,
6020                                                action,
6021                                                &tokens[tpos],
6022                                                n_tokens - tpos,
6023                                                instr,
6024                                                data);
6025
6026         if (!strcmp(tokens[tpos], "or"))
6027                 return instr_alu_or_translate(p,
6028                                               action,
6029                                               &tokens[tpos],
6030                                               n_tokens - tpos,
6031                                               instr,
6032                                               data);
6033
6034         if (!strcmp(tokens[tpos], "xor"))
6035                 return instr_alu_xor_translate(p,
6036                                                action,
6037                                                &tokens[tpos],
6038                                                n_tokens - tpos,
6039                                                instr,
6040                                                data);
6041
6042         if (!strcmp(tokens[tpos], "shl"))
6043                 return instr_alu_shl_translate(p,
6044                                                action,
6045                                                &tokens[tpos],
6046                                                n_tokens - tpos,
6047                                                instr,
6048                                                data);
6049
6050         if (!strcmp(tokens[tpos], "shr"))
6051                 return instr_alu_shr_translate(p,
6052                                                action,
6053                                                &tokens[tpos],
6054                                                n_tokens - tpos,
6055                                                instr,
6056                                                data);
6057
6058         if (!strcmp(tokens[tpos], "regprefetch"))
6059                 return instr_regprefetch_translate(p,
6060                                                    action,
6061                                                    &tokens[tpos],
6062                                                    n_tokens - tpos,
6063                                                    instr,
6064                                                    data);
6065
6066         if (!strcmp(tokens[tpos], "regrd"))
6067                 return instr_regrd_translate(p,
6068                                              action,
6069                                              &tokens[tpos],
6070                                              n_tokens - tpos,
6071                                              instr,
6072                                              data);
6073
6074         if (!strcmp(tokens[tpos], "regwr"))
6075                 return instr_regwr_translate(p,
6076                                              action,
6077                                              &tokens[tpos],
6078                                              n_tokens - tpos,
6079                                              instr,
6080                                              data);
6081
6082         if (!strcmp(tokens[tpos], "regadd"))
6083                 return instr_regadd_translate(p,
6084                                               action,
6085                                               &tokens[tpos],
6086                                               n_tokens - tpos,
6087                                               instr,
6088                                               data);
6089
6090         if (!strcmp(tokens[tpos], "metprefetch"))
6091                 return instr_metprefetch_translate(p,
6092                                                    action,
6093                                                    &tokens[tpos],
6094                                                    n_tokens - tpos,
6095                                                    instr,
6096                                                    data);
6097
6098         if (!strcmp(tokens[tpos], "meter"))
6099                 return instr_meter_translate(p,
6100                                              action,
6101                                              &tokens[tpos],
6102                                              n_tokens - tpos,
6103                                              instr,
6104                                              data);
6105
6106         if (!strcmp(tokens[tpos], "table"))
6107                 return instr_table_translate(p,
6108                                              action,
6109                                              &tokens[tpos],
6110                                              n_tokens - tpos,
6111                                              instr,
6112                                              data);
6113
6114         if (!strcmp(tokens[tpos], "learn"))
6115                 return instr_learn_translate(p,
6116                                              action,
6117                                              &tokens[tpos],
6118                                              n_tokens - tpos,
6119                                              instr,
6120                                              data);
6121         if (!strcmp(tokens[tpos], "rearm"))
6122                 return instr_rearm_translate(p,
6123                                              action,
6124                                              &tokens[tpos],
6125                                              n_tokens - tpos,
6126                                              instr,
6127                                              data);
6128
6129         if (!strcmp(tokens[tpos], "forget"))
6130                 return instr_forget_translate(p,
6131                                               action,
6132                                               &tokens[tpos],
6133                                               n_tokens - tpos,
6134                                               instr,
6135                                               data);
6136
6137         if (!strcmp(tokens[tpos], "extern"))
6138                 return instr_extern_translate(p,
6139                                               action,
6140                                               &tokens[tpos],
6141                                               n_tokens - tpos,
6142                                               instr,
6143                                               data);
6144
6145         if (!strcmp(tokens[tpos], "jmp"))
6146                 return instr_jmp_translate(p,
6147                                            action,
6148                                            &tokens[tpos],
6149                                            n_tokens - tpos,
6150                                            instr,
6151                                            data);
6152
6153         if (!strcmp(tokens[tpos], "jmpv"))
6154                 return instr_jmp_valid_translate(p,
6155                                                  action,
6156                                                  &tokens[tpos],
6157                                                  n_tokens - tpos,
6158                                                  instr,
6159                                                  data);
6160
6161         if (!strcmp(tokens[tpos], "jmpnv"))
6162                 return instr_jmp_invalid_translate(p,
6163                                                    action,
6164                                                    &tokens[tpos],
6165                                                    n_tokens - tpos,
6166                                                    instr,
6167                                                    data);
6168
6169         if (!strcmp(tokens[tpos], "jmph"))
6170                 return instr_jmp_hit_translate(p,
6171                                                action,
6172                                                &tokens[tpos],
6173                                                n_tokens - tpos,
6174                                                instr,
6175                                                data);
6176
6177         if (!strcmp(tokens[tpos], "jmpnh"))
6178                 return instr_jmp_miss_translate(p,
6179                                                 action,
6180                                                 &tokens[tpos],
6181                                                 n_tokens - tpos,
6182                                                 instr,
6183                                                 data);
6184
6185         if (!strcmp(tokens[tpos], "jmpa"))
6186                 return instr_jmp_action_hit_translate(p,
6187                                                       action,
6188                                                       &tokens[tpos],
6189                                                       n_tokens - tpos,
6190                                                       instr,
6191                                                       data);
6192
6193         if (!strcmp(tokens[tpos], "jmpna"))
6194                 return instr_jmp_action_miss_translate(p,
6195                                                        action,
6196                                                        &tokens[tpos],
6197                                                        n_tokens - tpos,
6198                                                        instr,
6199                                                        data);
6200
6201         if (!strcmp(tokens[tpos], "jmpeq"))
6202                 return instr_jmp_eq_translate(p,
6203                                               action,
6204                                               &tokens[tpos],
6205                                               n_tokens - tpos,
6206                                               instr,
6207                                               data);
6208
6209         if (!strcmp(tokens[tpos], "jmpneq"))
6210                 return instr_jmp_neq_translate(p,
6211                                                action,
6212                                                &tokens[tpos],
6213                                                n_tokens - tpos,
6214                                                instr,
6215                                                data);
6216
6217         if (!strcmp(tokens[tpos], "jmplt"))
6218                 return instr_jmp_lt_translate(p,
6219                                               action,
6220                                               &tokens[tpos],
6221                                               n_tokens - tpos,
6222                                               instr,
6223                                               data);
6224
6225         if (!strcmp(tokens[tpos], "jmpgt"))
6226                 return instr_jmp_gt_translate(p,
6227                                               action,
6228                                               &tokens[tpos],
6229                                               n_tokens - tpos,
6230                                               instr,
6231                                               data);
6232
6233         if (!strcmp(tokens[tpos], "return"))
6234                 return instr_return_translate(p,
6235                                               action,
6236                                               &tokens[tpos],
6237                                               n_tokens - tpos,
6238                                               instr,
6239                                               data);
6240
6241         return -EINVAL;
6242 }
6243
6244 static struct instruction_data *
6245 label_find(struct instruction_data *data, uint32_t n, const char *label)
6246 {
6247         uint32_t i;
6248
6249         for (i = 0; i < n; i++)
6250                 if (!strcmp(label, data[i].label))
6251                         return &data[i];
6252
6253         return NULL;
6254 }
6255
6256 static uint32_t
6257 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6258 {
6259         uint32_t count = 0, i;
6260
6261         if (!label[0])
6262                 return 0;
6263
6264         for (i = 0; i < n; i++)
6265                 if (!strcmp(label, data[i].jmp_label))
6266                         count++;
6267
6268         return count;
6269 }
6270
6271 static int
6272 instr_label_check(struct instruction_data *instruction_data,
6273                   uint32_t n_instructions)
6274 {
6275         uint32_t i;
6276
6277         /* Check that all instruction labels are unique. */
6278         for (i = 0; i < n_instructions; i++) {
6279                 struct instruction_data *data = &instruction_data[i];
6280                 char *label = data->label;
6281                 uint32_t j;
6282
6283                 if (!label[0])
6284                         continue;
6285
6286                 for (j = i + 1; j < n_instructions; j++)
6287                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6288         }
6289
6290         /* Check that no jump instruction (either conditional or not) can jump to itself (loop). */
6291         for (i = 0; i < n_instructions; i++) {
6292                 struct instruction_data *data = &instruction_data[i];
6293                 char *label = data->label;
6294                 char *jmp_label = data->jmp_label;
6295
6296                 /* Continue if this instruction does not have a label or it is not a jump. */
6297                 if (!label[0] || !jmp_label[0])
6298                         continue;
6299
6300                 CHECK(strcmp(label, jmp_label), EINVAL);
6301         }
6302
6303         /* Get users for each instruction label. */
6304         for (i = 0; i < n_instructions; i++) {
6305                 struct instruction_data *data = &instruction_data[i];
6306                 char *label = data->label;
6307
6308                 data->n_users = label_is_used(instruction_data,
6309                                               n_instructions,
6310                                               label);
6311         }
6312
6313         return 0;
6314 }
6315
6316 static int
6317 instr_jmp_resolve(struct instruction *instructions,
6318                   struct instruction_data *instruction_data,
6319                   uint32_t n_instructions)
6320 {
6321         uint32_t i;
6322
6323         for (i = 0; i < n_instructions; i++) {
6324                 struct instruction *instr = &instructions[i];
6325                 struct instruction_data *data = &instruction_data[i];
6326                 struct instruction_data *found;
6327
6328                 if (!instruction_is_jmp(instr))
6329                         continue;
6330
6331                 found = label_find(instruction_data,
6332                                    n_instructions,
6333                                    data->jmp_label);
6334                 CHECK(found, EINVAL);
6335
6336                 instr->jmp.ip = &instructions[found - instruction_data];
6337         }
6338
6339         return 0;
6340 }
6341
6342 static int
6343 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6344              struct action *a,
6345              struct instruction *instr,
6346              struct instruction_data *data __rte_unused,
6347              uint32_t n_instructions)
6348 {
6349         if (!a) {
6350                 enum instruction_type type;
6351                 uint32_t i;
6352
6353                 /* Check that the first instruction is rx. */
6354                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6355
6356                 /* Check that there is at least one tx instruction. */
6357                 for (i = 0; i < n_instructions; i++) {
6358                         type = instr[i].type;
6359
6360                         if (instruction_is_tx(type))
6361                                 break;
6362                 }
6363                 CHECK(i < n_instructions, EINVAL);
6364
6365                 /* Check that the last instruction is either tx or unconditional
6366                  * jump.
6367                  */
6368                 type = instr[n_instructions - 1].type;
6369                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6370         }
6371
6372         if (a) {
6373                 enum instruction_type type;
6374                 uint32_t i;
6375
6376                 /* Check that there is at least one return or tx instruction. */
6377                 for (i = 0; i < n_instructions; i++) {
6378                         type = instr[i].type;
6379
6380                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6381                                 break;
6382                 }
6383                 CHECK(i < n_instructions, EINVAL);
6384         }
6385
6386         return 0;
6387 }
6388
6389 static uint32_t
6390 instr_compact(struct instruction *instructions,
6391               struct instruction_data *instruction_data,
6392               uint32_t n_instructions)
6393 {
6394         uint32_t i, pos = 0;
6395
6396         /* Eliminate the invalid instructions that have been optimized out. */
6397         for (i = 0; i < n_instructions; i++) {
6398                 struct instruction *instr = &instructions[i];
6399                 struct instruction_data *data = &instruction_data[i];
6400
6401                 if (data->invalid)
6402                         continue;
6403
6404                 if (i != pos) {
6405                         memcpy(&instructions[pos], instr, sizeof(*instr));
6406                         memcpy(&instruction_data[pos], data, sizeof(*data));
6407                 }
6408
6409                 pos++;
6410         }
6411
6412         return pos;
6413 }
6414
6415 static int
6416 instr_pattern_extract_many_search(struct instruction *instr,
6417                                   struct instruction_data *data,
6418                                   uint32_t n_instr,
6419                                   uint32_t *n_pattern_instr)
6420 {
6421         uint32_t i;
6422
6423         for (i = 0; i < n_instr; i++) {
6424                 if (data[i].invalid)
6425                         break;
6426
6427                 if (instr[i].type != INSTR_HDR_EXTRACT)
6428                         break;
6429
6430                 if (i == RTE_DIM(instr->io.hdr.header_id))
6431                         break;
6432
6433                 if (i && data[i].n_users)
6434                         break;
6435         }
6436
6437         if (i < 2)
6438                 return 0;
6439
6440         *n_pattern_instr = i;
6441         return 1;
6442 }
6443
6444 static void
6445 instr_pattern_extract_many_replace(struct instruction *instr,
6446                                    struct instruction_data *data,
6447                                    uint32_t n_instr)
6448 {
6449         uint32_t i;
6450
6451         for (i = 1; i < n_instr; i++) {
6452                 instr[0].type++;
6453                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6454                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6455                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6456
6457                 data[i].invalid = 1;
6458         }
6459 }
6460
6461 static uint32_t
6462 instr_pattern_extract_many_optimize(struct instruction *instructions,
6463                                     struct instruction_data *instruction_data,
6464                                     uint32_t n_instructions)
6465 {
6466         uint32_t i;
6467
6468         for (i = 0; i < n_instructions; ) {
6469                 struct instruction *instr = &instructions[i];
6470                 struct instruction_data *data = &instruction_data[i];
6471                 uint32_t n_instr = 0;
6472                 int detected;
6473
6474                 /* Extract many. */
6475                 detected = instr_pattern_extract_many_search(instr,
6476                                                              data,
6477                                                              n_instructions - i,
6478                                                              &n_instr);
6479                 if (detected) {
6480                         instr_pattern_extract_many_replace(instr,
6481                                                            data,
6482                                                            n_instr);
6483                         i += n_instr;
6484                         continue;
6485                 }
6486
6487                 /* No pattern starting at the current instruction. */
6488                 i++;
6489         }
6490
6491         /* Eliminate the invalid instructions that have been optimized out. */
6492         n_instructions = instr_compact(instructions,
6493                                        instruction_data,
6494                                        n_instructions);
6495
6496         return n_instructions;
6497 }
6498
6499 static int
6500 instr_pattern_emit_many_tx_search(struct instruction *instr,
6501                                   struct instruction_data *data,
6502                                   uint32_t n_instr,
6503                                   uint32_t *n_pattern_instr)
6504 {
6505         uint32_t i;
6506
6507         for (i = 0; i < n_instr; i++) {
6508                 if (data[i].invalid)
6509                         break;
6510
6511                 if (instr[i].type != INSTR_HDR_EMIT)
6512                         break;
6513
6514                 if (i == RTE_DIM(instr->io.hdr.header_id))
6515                         break;
6516
6517                 if (i && data[i].n_users)
6518                         break;
6519         }
6520
6521         if (!i)
6522                 return 0;
6523
6524         if (instr[i].type != INSTR_TX)
6525                 return 0;
6526
6527         if (data[i].n_users)
6528                 return 0;
6529
6530         i++;
6531
6532         *n_pattern_instr = i;
6533         return 1;
6534 }
6535
6536 static void
6537 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6538                                    struct instruction_data *data,
6539                                    uint32_t n_instr)
6540 {
6541         uint32_t i;
6542
6543         /* Any emit instruction in addition to the first one. */
6544         for (i = 1; i < n_instr - 1; i++) {
6545                 instr[0].type++;
6546                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6547                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6548                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6549
6550                 data[i].invalid = 1;
6551         }
6552
6553         /* The TX instruction is the last one in the pattern. */
6554         instr[0].type++;
6555         instr[0].io.io.offset = instr[i].io.io.offset;
6556         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6557         data[i].invalid = 1;
6558 }
6559
6560 static uint32_t
6561 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6562                                     struct instruction_data *instruction_data,
6563                                     uint32_t n_instructions)
6564 {
6565         uint32_t i;
6566
6567         for (i = 0; i < n_instructions; ) {
6568                 struct instruction *instr = &instructions[i];
6569                 struct instruction_data *data = &instruction_data[i];
6570                 uint32_t n_instr = 0;
6571                 int detected;
6572
6573                 /* Emit many + TX. */
6574                 detected = instr_pattern_emit_many_tx_search(instr,
6575                                                              data,
6576                                                              n_instructions - i,
6577                                                              &n_instr);
6578                 if (detected) {
6579                         instr_pattern_emit_many_tx_replace(instr,
6580                                                            data,
6581                                                            n_instr);
6582                         i += n_instr;
6583                         continue;
6584                 }
6585
6586                 /* No pattern starting at the current instruction. */
6587                 i++;
6588         }
6589
6590         /* Eliminate the invalid instructions that have been optimized out. */
6591         n_instructions = instr_compact(instructions,
6592                                        instruction_data,
6593                                        n_instructions);
6594
6595         return n_instructions;
6596 }
6597
6598 static uint32_t
6599 action_arg_src_mov_count(struct action *a,
6600                          uint32_t arg_id,
6601                          struct instruction *instructions,
6602                          struct instruction_data *instruction_data,
6603                          uint32_t n_instructions);
6604
6605 static int
6606 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6607                                       struct action *a,
6608                                       struct instruction *instr,
6609                                       struct instruction_data *data,
6610                                       uint32_t n_instr,
6611                                       struct instruction *instructions,
6612                                       struct instruction_data *instruction_data,
6613                                       uint32_t n_instructions,
6614                                       uint32_t *n_pattern_instr)
6615 {
6616         struct header *h;
6617         uint32_t src_field_id, i, j;
6618
6619         /* Prerequisites. */
6620         if (!a || !a->st)
6621                 return 0;
6622
6623         /* First instruction: MOV_HM. */
6624         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6625                 return 0;
6626
6627         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6628         if (!h || h->st->var_size)
6629                 return 0;
6630
6631         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6632                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6633                         break;
6634
6635         if (src_field_id == a->st->n_fields)
6636                 return 0;
6637
6638         if (instr[0].mov.dst.offset ||
6639             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6640             instr[0].mov.src.struct_id ||
6641             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6642             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6643                 return 0;
6644
6645         if ((n_instr < h->st->n_fields + 1) ||
6646              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6647                 return 0;
6648
6649         /* Subsequent instructions: MOV_HM. */
6650         for (i = 1; i < h->st->n_fields; i++)
6651                 if (data[i].invalid ||
6652                     data[i].n_users ||
6653                     (instr[i].type != INSTR_MOV_HM) ||
6654                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6655                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6656                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6657                     instr[i].mov.src.struct_id ||
6658                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6659                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6660                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6661                         return 0;
6662
6663         /* Last instruction: HDR_VALIDATE. */
6664         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6665             (instr[i].valid.header_id != h->id))
6666                 return 0;
6667
6668         /* Check that none of the action args that are used as source for this
6669          * DMA transfer are not used as source in any other mov instruction.
6670          */
6671         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6672                 uint32_t n_users;
6673
6674                 n_users = action_arg_src_mov_count(a,
6675                                                    j,
6676                                                    instructions,
6677                                                    instruction_data,
6678                                                    n_instructions);
6679                 if (n_users > 1)
6680                         return 0;
6681         }
6682
6683         *n_pattern_instr = 1 + i;
6684         return 1;
6685 }
6686
6687 static void
6688 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6689                                        struct action *a,
6690                                        struct instruction *instr,
6691                                        struct instruction_data *data,
6692                                        uint32_t n_instr)
6693 {
6694         struct header *h;
6695         uint32_t src_field_id, src_offset, i;
6696
6697         /* Read from the instructions before they are modified. */
6698         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6699         if (!h)
6700                 return;
6701
6702         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6703                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6704                         break;
6705
6706         if (src_field_id == a->st->n_fields)
6707                 return;
6708
6709         src_offset = instr[0].mov.src.offset;
6710
6711         /* Modify the instructions. */
6712         instr[0].type = INSTR_DMA_HT;
6713         instr[0].dma.dst.header_id[0] = h->id;
6714         instr[0].dma.dst.struct_id[0] = h->struct_id;
6715         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6716         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6717
6718         for (i = 1; i < n_instr; i++)
6719                 data[i].invalid = 1;
6720
6721         /* Update the endianness of the action arguments to header endianness. */
6722         for (i = 0; i < h->st->n_fields; i++)
6723                 a->args_endianness[src_field_id + i] = 1;
6724 }
6725
6726 static uint32_t
6727 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6728                                         struct action *a,
6729                                         struct instruction *instructions,
6730                                         struct instruction_data *instruction_data,
6731                                         uint32_t n_instructions)
6732 {
6733         uint32_t i;
6734
6735         if (!a || !a->st)
6736                 return n_instructions;
6737
6738         for (i = 0; i < n_instructions; ) {
6739                 struct instruction *instr = &instructions[i];
6740                 struct instruction_data *data = &instruction_data[i];
6741                 uint32_t n_instr = 0;
6742                 int detected;
6743
6744                 /* Mov all + validate. */
6745                 detected = instr_pattern_mov_all_validate_search(p,
6746                                                                  a,
6747                                                                  instr,
6748                                                                  data,
6749                                                                  n_instructions - i,
6750                                                                  instructions,
6751                                                                  instruction_data,
6752                                                                  n_instructions,
6753                                                                  &n_instr);
6754                 if (detected) {
6755                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6756                         i += n_instr;
6757                         continue;
6758                 }
6759
6760                 /* No pattern starting at the current instruction. */
6761                 i++;
6762         }
6763
6764         /* Eliminate the invalid instructions that have been optimized out. */
6765         n_instructions = instr_compact(instructions,
6766                                        instruction_data,
6767                                        n_instructions);
6768
6769         return n_instructions;
6770 }
6771
6772 static int
6773 instr_pattern_dma_many_search(struct instruction *instr,
6774                               struct instruction_data *data,
6775                               uint32_t n_instr,
6776                               uint32_t *n_pattern_instr)
6777 {
6778         uint32_t i;
6779
6780         for (i = 0; i < n_instr; i++) {
6781                 if (data[i].invalid)
6782                         break;
6783
6784                 if (instr[i].type != INSTR_DMA_HT)
6785                         break;
6786
6787                 if (i == RTE_DIM(instr->dma.dst.header_id))
6788                         break;
6789
6790                 if (i && data[i].n_users)
6791                         break;
6792         }
6793
6794         if (i < 2)
6795                 return 0;
6796
6797         *n_pattern_instr = i;
6798         return 1;
6799 }
6800
6801 static void
6802 instr_pattern_dma_many_replace(struct instruction *instr,
6803                                struct instruction_data *data,
6804                                uint32_t n_instr)
6805 {
6806         uint32_t i;
6807
6808         for (i = 1; i < n_instr; i++) {
6809                 instr[0].type++;
6810                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6811                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6812                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6813                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6814
6815                 data[i].invalid = 1;
6816         }
6817 }
6818
6819 static uint32_t
6820 instr_pattern_dma_many_optimize(struct instruction *instructions,
6821                struct instruction_data *instruction_data,
6822                uint32_t n_instructions)
6823 {
6824         uint32_t i;
6825
6826         for (i = 0; i < n_instructions; ) {
6827                 struct instruction *instr = &instructions[i];
6828                 struct instruction_data *data = &instruction_data[i];
6829                 uint32_t n_instr = 0;
6830                 int detected;
6831
6832                 /* DMA many. */
6833                 detected = instr_pattern_dma_many_search(instr,
6834                                                          data,
6835                                                          n_instructions - i,
6836                                                          &n_instr);
6837                 if (detected) {
6838                         instr_pattern_dma_many_replace(instr, data, n_instr);
6839                         i += n_instr;
6840                         continue;
6841                 }
6842
6843                 /* No pattern starting at the current instruction. */
6844                 i++;
6845         }
6846
6847         /* Eliminate the invalid instructions that have been optimized out. */
6848         n_instructions = instr_compact(instructions,
6849                                        instruction_data,
6850                                        n_instructions);
6851
6852         return n_instructions;
6853 }
6854
6855 static uint32_t
6856 instr_optimize(struct rte_swx_pipeline *p,
6857                struct action *a,
6858                struct instruction *instructions,
6859                struct instruction_data *instruction_data,
6860                uint32_t n_instructions)
6861 {
6862         /* Extract many. */
6863         n_instructions = instr_pattern_extract_many_optimize(instructions,
6864                                                              instruction_data,
6865                                                              n_instructions);
6866
6867         /* Emit many + TX. */
6868         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6869                                                              instruction_data,
6870                                                              n_instructions);
6871
6872         /* Mov all + validate. */
6873         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6874                                                                  a,
6875                                                                  instructions,
6876                                                                  instruction_data,
6877                                                                  n_instructions);
6878
6879         /* DMA many. */
6880         n_instructions = instr_pattern_dma_many_optimize(instructions,
6881                                                          instruction_data,
6882                                                          n_instructions);
6883
6884         return n_instructions;
6885 }
6886
6887 static int
6888 instruction_config(struct rte_swx_pipeline *p,
6889                    struct action *a,
6890                    const char **instructions,
6891                    uint32_t n_instructions)
6892 {
6893         struct instruction *instr = NULL;
6894         struct instruction_data *data = NULL;
6895         int err = 0;
6896         uint32_t i;
6897
6898         CHECK(n_instructions, EINVAL);
6899         CHECK(instructions, EINVAL);
6900         for (i = 0; i < n_instructions; i++)
6901                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6902
6903         /* Memory allocation. */
6904         instr = calloc(n_instructions, sizeof(struct instruction));
6905         if (!instr) {
6906                 err = -ENOMEM;
6907                 goto error;
6908         }
6909
6910         data = calloc(n_instructions, sizeof(struct instruction_data));
6911         if (!data) {
6912                 err = -ENOMEM;
6913                 goto error;
6914         }
6915
6916         for (i = 0; i < n_instructions; i++) {
6917                 char *string = strdup(instructions[i]);
6918                 if (!string) {
6919                         err = -ENOMEM;
6920                         goto error;
6921                 }
6922
6923                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6924                 if (err) {
6925                         free(string);
6926                         goto error;
6927                 }
6928
6929                 free(string);
6930         }
6931
6932         err = instr_label_check(data, n_instructions);
6933         if (err)
6934                 goto error;
6935
6936         err = instr_verify(p, a, instr, data, n_instructions);
6937         if (err)
6938                 goto error;
6939
6940         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6941
6942         err = instr_jmp_resolve(instr, data, n_instructions);
6943         if (err)
6944                 goto error;
6945
6946         if (a) {
6947                 a->instructions = instr;
6948                 a->instruction_data = data;
6949                 a->n_instructions = n_instructions;
6950         } else {
6951                 p->instructions = instr;
6952                 p->instruction_data = data;
6953                 p->n_instructions = n_instructions;
6954         }
6955
6956         return 0;
6957
6958 error:
6959         free(data);
6960         free(instr);
6961         return err;
6962 }
6963
6964 static instr_exec_t instruction_table[] = {
6965         [INSTR_RX] = instr_rx_exec,
6966         [INSTR_TX] = instr_tx_exec,
6967         [INSTR_TX_I] = instr_tx_i_exec,
6968         [INSTR_DROP] = instr_drop_exec,
6969         [INSTR_MIRROR] = instr_mirror_exec,
6970         [INSTR_RECIRCULATE] = instr_recirculate_exec,
6971         [INSTR_RECIRCID] = instr_recircid_exec,
6972
6973         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6974         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6975         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6976         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6977         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6978         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6979         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6980         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6981         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6982         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6983
6984         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6985         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6986         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6987         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6988         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6989         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6990         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6991         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6992         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6993
6994         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6995         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6996
6997         [INSTR_MOV] = instr_mov_exec,
6998         [INSTR_MOV_MH] = instr_mov_mh_exec,
6999         [INSTR_MOV_HM] = instr_mov_hm_exec,
7000         [INSTR_MOV_HH] = instr_mov_hh_exec,
7001         [INSTR_MOV_I] = instr_mov_i_exec,
7002
7003         [INSTR_DMA_HT] = instr_dma_ht_exec,
7004         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7005         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7006         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7007         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7008         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7009         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7010         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7011
7012         [INSTR_ALU_ADD] = instr_alu_add_exec,
7013         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7014         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7015         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7016         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7017         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7018
7019         [INSTR_ALU_SUB] = instr_alu_sub_exec,
7020         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7021         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7022         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7023         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7024         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7025
7026         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7027         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7028         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7029         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7030
7031         [INSTR_ALU_AND] = instr_alu_and_exec,
7032         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7033         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7034         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7035         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7036
7037         [INSTR_ALU_OR] = instr_alu_or_exec,
7038         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7039         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7040         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7041         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7042
7043         [INSTR_ALU_XOR] = instr_alu_xor_exec,
7044         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7045         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7046         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7047         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7048
7049         [INSTR_ALU_SHL] = instr_alu_shl_exec,
7050         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7051         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7052         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7053         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7054         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7055
7056         [INSTR_ALU_SHR] = instr_alu_shr_exec,
7057         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7058         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7059         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7060         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7061         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7062
7063         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7064         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7065         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7066
7067         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7068         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7069         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7070         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7071         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7072         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7073
7074         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7075         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7076         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7077         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7078         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7079         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7080         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7081         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7082         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7083
7084         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7085         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7086         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7087         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7088         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7089         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7090         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7091         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7092         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7093
7094         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7095         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7096         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7097
7098         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7099         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7100         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7101         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7102         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7103         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7104         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7105         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7106         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7107         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7108         [INSTR_METER_IMM] = instr_meter_imm_exec,
7109         [INSTR_METER_IMI] = instr_meter_imi_exec,
7110
7111         [INSTR_TABLE] = instr_table_exec,
7112         [INSTR_TABLE_AF] = instr_table_af_exec,
7113         [INSTR_SELECTOR] = instr_selector_exec,
7114         [INSTR_LEARNER] = instr_learner_exec,
7115         [INSTR_LEARNER_AF] = instr_learner_af_exec,
7116         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7117         [INSTR_LEARNER_REARM] = instr_rearm_exec,
7118         [INSTR_LEARNER_REARM_NEW] = instr_rearm_new_exec,
7119         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7120         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7121         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7122
7123         [INSTR_JMP] = instr_jmp_exec,
7124         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7125         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7126         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7127         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7128         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7129         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7130
7131         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7132         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7133         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7134         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7135         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7136
7137         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7138         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7139         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7140         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7141         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7142
7143         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7144         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7145         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7146         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7147         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7148         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7149
7150         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7151         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7152         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7153         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7154         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7155         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7156
7157         [INSTR_RETURN] = instr_return_exec,
7158 };
7159
7160 static int
7161 instruction_table_build(struct rte_swx_pipeline *p)
7162 {
7163         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
7164                                       sizeof(struct instr_exec_t *));
7165         if (!p->instruction_table)
7166                 return -EINVAL;
7167
7168         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
7169
7170         return 0;
7171 }
7172
7173 static void
7174 instruction_table_build_free(struct rte_swx_pipeline *p)
7175 {
7176         if (!p->instruction_table)
7177                 return;
7178
7179         free(p->instruction_table);
7180         p->instruction_table = NULL;
7181 }
7182
7183 static void
7184 instruction_table_free(struct rte_swx_pipeline *p)
7185 {
7186         instruction_table_build_free(p);
7187 }
7188
7189 static inline void
7190 instr_exec(struct rte_swx_pipeline *p)
7191 {
7192         struct thread *t = &p->threads[p->thread_id];
7193         struct instruction *ip = t->ip;
7194         instr_exec_t instr = p->instruction_table[ip->type];
7195
7196         instr(p);
7197 }
7198
7199 /*
7200  * Action.
7201  */
7202 static struct action *
7203 action_find(struct rte_swx_pipeline *p, const char *name)
7204 {
7205         struct action *elem;
7206
7207         if (!name)
7208                 return NULL;
7209
7210         TAILQ_FOREACH(elem, &p->actions, node)
7211                 if (strcmp(elem->name, name) == 0)
7212                         return elem;
7213
7214         return NULL;
7215 }
7216
7217 static struct action *
7218 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7219 {
7220         struct action *action = NULL;
7221
7222         TAILQ_FOREACH(action, &p->actions, node)
7223                 if (action->id == id)
7224                         return action;
7225
7226         return NULL;
7227 }
7228
7229 static struct field *
7230 action_field_find(struct action *a, const char *name)
7231 {
7232         return a->st ? struct_type_field_find(a->st, name) : NULL;
7233 }
7234
7235 static struct field *
7236 action_field_parse(struct action *action, const char *name)
7237 {
7238         if (name[0] != 't' || name[1] != '.')
7239                 return NULL;
7240
7241         return action_field_find(action, &name[2]);
7242 }
7243
7244 static int
7245 action_has_nbo_args(struct action *a)
7246 {
7247         uint32_t i;
7248
7249         /* Return if the action does not have any args. */
7250         if (!a->st)
7251                 return 0; /* FALSE */
7252
7253         for (i = 0; i < a->st->n_fields; i++)
7254                 if (a->args_endianness[i])
7255                         return 1; /* TRUE */
7256
7257         return 0; /* FALSE */
7258 }
7259
7260 static int
7261 action_does_learning(struct action *a)
7262 {
7263         uint32_t i;
7264
7265         for (i = 0; i < a->n_instructions; i++)
7266                 switch (a->instructions[i].type) {
7267                 case INSTR_LEARNER_LEARN:
7268                         return 1; /* TRUE */
7269
7270                 case INSTR_LEARNER_FORGET:
7271                         return 1; /* TRUE */
7272
7273                 default:
7274                         continue;
7275                 }
7276
7277         return 0; /* FALSE */
7278 }
7279
7280 int
7281 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7282                                const char *name,
7283                                const char *args_struct_type_name,
7284                                const char **instructions,
7285                                uint32_t n_instructions)
7286 {
7287         struct struct_type *args_struct_type = NULL;
7288         struct action *a = NULL;
7289         int status = 0;
7290
7291         CHECK(p, EINVAL);
7292
7293         CHECK_NAME(name, EINVAL);
7294         CHECK(!action_find(p, name), EEXIST);
7295
7296         if (args_struct_type_name) {
7297                 CHECK_NAME(args_struct_type_name, EINVAL);
7298                 args_struct_type = struct_type_find(p, args_struct_type_name);
7299                 CHECK(args_struct_type, EINVAL);
7300                 CHECK(!args_struct_type->var_size, EINVAL);
7301         }
7302
7303         /* Node allocation. */
7304         a = calloc(1, sizeof(struct action));
7305         if (!a) {
7306                 status = -ENOMEM;
7307                 goto error;
7308         }
7309
7310         if (args_struct_type) {
7311                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7312                 if (!a->args_endianness) {
7313                         status = -ENOMEM;
7314                         goto error;
7315                 }
7316         }
7317
7318         /* Node initialization. */
7319         strcpy(a->name, name);
7320         a->st = args_struct_type;
7321         a->id = p->n_actions;
7322
7323         /* Instruction translation. */
7324         status = instruction_config(p, a, instructions, n_instructions);
7325         if (status)
7326                 goto error;
7327
7328         /* Node add to tailq. */
7329         TAILQ_INSERT_TAIL(&p->actions, a, node);
7330         p->n_actions++;
7331
7332         return 0;
7333
7334 error:
7335         if (!a)
7336                 return status;
7337
7338         free(a->args_endianness);
7339         free(a->instructions);
7340         free(a->instruction_data);
7341         free(a);
7342
7343         return status;
7344 }
7345
7346 static int
7347 action_build(struct rte_swx_pipeline *p)
7348 {
7349         struct action *action;
7350
7351         /* p->action_instructions. */
7352         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7353         CHECK(p->action_instructions, ENOMEM);
7354
7355         TAILQ_FOREACH(action, &p->actions, node)
7356                 p->action_instructions[action->id] = action->instructions;
7357
7358         /* p->action_funcs. */
7359         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7360         CHECK(p->action_funcs, ENOMEM);
7361
7362         return 0;
7363 }
7364
7365 static void
7366 action_build_free(struct rte_swx_pipeline *p)
7367 {
7368         free(p->action_funcs);
7369         p->action_funcs = NULL;
7370
7371         free(p->action_instructions);
7372         p->action_instructions = NULL;
7373 }
7374
7375 static void
7376 action_free(struct rte_swx_pipeline *p)
7377 {
7378         action_build_free(p);
7379
7380         for ( ; ; ) {
7381                 struct action *action;
7382
7383                 action = TAILQ_FIRST(&p->actions);
7384                 if (!action)
7385                         break;
7386
7387                 TAILQ_REMOVE(&p->actions, action, node);
7388                 free(action->args_endianness);
7389                 free(action->instructions);
7390                 free(action->instruction_data);
7391                 free(action);
7392         }
7393 }
7394
7395 static uint32_t
7396 action_arg_src_mov_count(struct action *a,
7397                          uint32_t arg_id,
7398                          struct instruction *instructions,
7399                          struct instruction_data *instruction_data,
7400                          uint32_t n_instructions)
7401 {
7402         uint32_t offset, n_users = 0, i;
7403
7404         if (!a->st ||
7405             (arg_id >= a->st->n_fields) ||
7406             !instructions ||
7407             !instruction_data ||
7408             !n_instructions)
7409                 return 0;
7410
7411         offset = a->st->fields[arg_id].offset / 8;
7412
7413         for (i = 0; i < n_instructions; i++) {
7414                 struct instruction *instr = &instructions[i];
7415                 struct instruction_data *data = &instruction_data[i];
7416
7417                 if (data->invalid ||
7418                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7419                     instr->mov.src.struct_id ||
7420                     (instr->mov.src.offset != offset))
7421                         continue;
7422
7423                 n_users++;
7424         }
7425
7426         return n_users;
7427 }
7428
7429 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7430 #define field_ntoh(val, n_bits) (ntoh64((val) << (64 - n_bits)))
7431 #define field_hton(val, n_bits) (hton64((val) << (64 - n_bits)))
7432 #else
7433 #define field_ntoh(val, n_bits) (val)
7434 #define field_hton(val, n_bits) (val)
7435 #endif
7436
7437 #define ACTION_ARGS_TOKENS_MAX 256
7438
7439 static int
7440 action_args_parse(struct action *a, const char *args, uint8_t *data)
7441 {
7442         char *tokens[ACTION_ARGS_TOKENS_MAX], *s0 = NULL, *s;
7443         uint32_t n_tokens = 0, offset = 0, i;
7444         int status = 0;
7445
7446         /* Checks. */
7447         if (!a->st || !args || !args[0]) {
7448                 status = -EINVAL;
7449                 goto error;
7450         }
7451
7452         /* Memory allocation. */
7453         s0 = strdup(args);
7454         if (!s0) {
7455                 status = -ENOMEM;
7456                 goto error;
7457         }
7458
7459         /* Parse the string into tokens. */
7460         for (s = s0; ; ) {
7461                 char *token;
7462
7463                 token = strtok_r(s, " \f\n\r\t\v", &s);
7464                 if (!token)
7465                         break;
7466
7467                 if (n_tokens >= RTE_DIM(tokens)) {
7468                         status = -EINVAL;
7469                         goto error;
7470                 }
7471
7472                 tokens[n_tokens] = token;
7473                 n_tokens++;
7474         }
7475
7476         /* More checks. */
7477         if (n_tokens != a->st->n_fields * 2) {
7478                 status = -EINVAL;
7479                 goto error;
7480         }
7481
7482         /* Process the action arguments. */
7483         for (i = 0; i < a->st->n_fields; i++) {
7484                 struct field *f = &a->st->fields[i];
7485                 char *arg_name = tokens[i * 2];
7486                 char *arg_val = tokens[i * 2 + 1];
7487                 uint64_t val;
7488
7489                 if (strcmp(arg_name, f->name)) {
7490                         status = -EINVAL;
7491                         goto error;
7492                 }
7493
7494                 val = strtoull(arg_val, &arg_val, 0);
7495                 if (arg_val[0]) {
7496                         status = -EINVAL;
7497                         goto error;
7498                 }
7499
7500                 /* Endianness conversion. */
7501                 if (a->args_endianness[i])
7502                         val = field_hton(val, f->n_bits);
7503
7504                 /* Copy to entry. */
7505                 memcpy(&data[offset], (uint8_t *)&val, f->n_bits / 8);
7506                 offset += f->n_bits / 8;
7507         }
7508
7509 error:
7510         free(s0);
7511         return status;
7512 }
7513
7514 /*
7515  * Table.
7516  */
7517 static struct table_type *
7518 table_type_find(struct rte_swx_pipeline *p, const char *name)
7519 {
7520         struct table_type *elem;
7521
7522         TAILQ_FOREACH(elem, &p->table_types, node)
7523                 if (strcmp(elem->name, name) == 0)
7524                         return elem;
7525
7526         return NULL;
7527 }
7528
7529 static struct table_type *
7530 table_type_resolve(struct rte_swx_pipeline *p,
7531                    const char *recommended_type_name,
7532                    enum rte_swx_table_match_type match_type)
7533 {
7534         struct table_type *elem;
7535
7536         /* Only consider the recommended type if the match type is correct. */
7537         if (recommended_type_name)
7538                 TAILQ_FOREACH(elem, &p->table_types, node)
7539                         if (!strcmp(elem->name, recommended_type_name) &&
7540                             (elem->match_type == match_type))
7541                                 return elem;
7542
7543         /* Ignore the recommended type and get the first element with this match
7544          * type.
7545          */
7546         TAILQ_FOREACH(elem, &p->table_types, node)
7547                 if (elem->match_type == match_type)
7548                         return elem;
7549
7550         return NULL;
7551 }
7552
7553 static struct table *
7554 table_find(struct rte_swx_pipeline *p, const char *name)
7555 {
7556         struct table *elem;
7557
7558         TAILQ_FOREACH(elem, &p->tables, node)
7559                 if (strcmp(elem->name, name) == 0)
7560                         return elem;
7561
7562         return NULL;
7563 }
7564
7565 static struct table *
7566 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7567 {
7568         struct table *table = NULL;
7569
7570         TAILQ_FOREACH(table, &p->tables, node)
7571                 if (table->id == id)
7572                         return table;
7573
7574         return NULL;
7575 }
7576
7577 int
7578 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7579                                      const char *name,
7580                                      enum rte_swx_table_match_type match_type,
7581                                      struct rte_swx_table_ops *ops)
7582 {
7583         struct table_type *elem;
7584
7585         CHECK(p, EINVAL);
7586
7587         CHECK_NAME(name, EINVAL);
7588         CHECK(!table_type_find(p, name), EEXIST);
7589
7590         CHECK(ops, EINVAL);
7591         CHECK(ops->create, EINVAL);
7592         CHECK(ops->lkp, EINVAL);
7593         CHECK(ops->free, EINVAL);
7594
7595         /* Node allocation. */
7596         elem = calloc(1, sizeof(struct table_type));
7597         CHECK(elem, ENOMEM);
7598
7599         /* Node initialization. */
7600         strcpy(elem->name, name);
7601         elem->match_type = match_type;
7602         memcpy(&elem->ops, ops, sizeof(*ops));
7603
7604         /* Node add to tailq. */
7605         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7606
7607         return 0;
7608 }
7609
7610 static int
7611 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7612                          uint32_t n_fields,
7613                          enum rte_swx_table_match_type *match_type)
7614 {
7615         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7616
7617         for (i = 0; i < n_fields; i++) {
7618                 struct rte_swx_match_field_params  *f = &fields[i];
7619
7620                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7621                         n_fields_em++;
7622
7623                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7624                         n_fields_lpm++;
7625         }
7626
7627         if ((n_fields_lpm > 1) ||
7628             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7629                 return -EINVAL;
7630
7631         *match_type = (n_fields_em == n_fields) ?
7632                        RTE_SWX_TABLE_MATCH_EXACT :
7633                        RTE_SWX_TABLE_MATCH_WILDCARD;
7634
7635         return 0;
7636 }
7637
7638 static int
7639 table_match_fields_check(struct rte_swx_pipeline *p,
7640                          struct rte_swx_pipeline_table_params *params,
7641                          struct header **header)
7642 {
7643         struct header *h0 = NULL;
7644         struct field *hf, *mf;
7645         uint32_t *offset = NULL, i;
7646         int status = 0;
7647
7648         /* Return if no match fields. */
7649         if (!params->n_fields) {
7650                 if (params->fields) {
7651                         status = -EINVAL;
7652                         goto end;
7653                 }
7654
7655                 if (header)
7656                         *header = NULL;
7657
7658                 return 0;
7659         }
7660
7661         /* Memory allocation. */
7662         offset = calloc(params->n_fields, sizeof(uint32_t));
7663         if (!offset) {
7664                 status = -ENOMEM;
7665                 goto end;
7666         }
7667
7668         /* Check that all the match fields belong to either the same header or
7669          * to the meta-data.
7670          */
7671         hf = header_field_parse(p, params->fields[0].name, &h0);
7672         mf = metadata_field_parse(p, params->fields[0].name);
7673         if ((!hf && !mf) || (hf && hf->var_size)) {
7674                 status = -EINVAL;
7675                 goto end;
7676         }
7677
7678         offset[0] = h0 ? hf->offset : mf->offset;
7679
7680         for (i = 1; i < params->n_fields; i++)
7681                 if (h0) {
7682                         struct header *h;
7683
7684                         hf = header_field_parse(p, params->fields[i].name, &h);
7685                         if (!hf || (h->id != h0->id) || hf->var_size) {
7686                                 status = -EINVAL;
7687                                 goto end;
7688                         }
7689
7690                         offset[i] = hf->offset;
7691                 } else {
7692                         mf = metadata_field_parse(p, params->fields[i].name);
7693                         if (!mf) {
7694                                 status = -EINVAL;
7695                                 goto end;
7696                         }
7697
7698                         offset[i] = mf->offset;
7699                 }
7700
7701         /* Check that there are no duplicated match fields. */
7702         for (i = 0; i < params->n_fields; i++) {
7703                 uint32_t j;
7704
7705                 for (j = 0; j < i; j++)
7706                         if (offset[j] == offset[i]) {
7707                                 status = -EINVAL;
7708                                 goto end;
7709                         }
7710         }
7711
7712         /* Return. */
7713         if (header)
7714                 *header = h0;
7715
7716 end:
7717         free(offset);
7718         return status;
7719 }
7720
7721 int
7722 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7723                               const char *name,
7724                               struct rte_swx_pipeline_table_params *params,
7725                               const char *recommended_table_type_name,
7726                               const char *args,
7727                               uint32_t size)
7728 {
7729         struct table_type *type;
7730         struct table *t = NULL;
7731         struct action *default_action;
7732         struct header *header = NULL;
7733         uint32_t action_data_size_max = 0, i;
7734         int status = 0;
7735
7736         CHECK(p, EINVAL);
7737
7738         CHECK_NAME(name, EINVAL);
7739         CHECK(!table_find(p, name), EEXIST);
7740         CHECK(!selector_find(p, name), EEXIST);
7741         CHECK(!learner_find(p, name), EEXIST);
7742
7743         CHECK(params, EINVAL);
7744
7745         /* Match checks. */
7746         status = table_match_fields_check(p, params, &header);
7747         if (status)
7748                 return status;
7749
7750         /* Action checks. */
7751         CHECK(params->n_actions, EINVAL);
7752         CHECK(params->action_names, EINVAL);
7753         for (i = 0; i < params->n_actions; i++) {
7754                 const char *action_name = params->action_names[i];
7755                 struct action *a;
7756                 uint32_t action_data_size;
7757                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7758
7759                 CHECK_NAME(action_name, EINVAL);
7760
7761                 a = action_find(p, action_name);
7762                 CHECK(a, EINVAL);
7763                 CHECK(!action_does_learning(a), EINVAL);
7764
7765                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7766                 if (action_data_size > action_data_size_max)
7767                         action_data_size_max = action_data_size;
7768
7769                 if (params->action_is_for_table_entries)
7770                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7771                 if (params->action_is_for_default_entry)
7772                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7773                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7774         }
7775
7776         CHECK_NAME(params->default_action_name, EINVAL);
7777         for (i = 0; i < p->n_actions; i++)
7778                 if (!strcmp(params->action_names[i],
7779                             params->default_action_name))
7780                         break;
7781         CHECK(i < params->n_actions, EINVAL);
7782         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7783               EINVAL);
7784
7785         default_action = action_find(p, params->default_action_name);
7786         CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
7787               EINVAL);
7788
7789         /* Table type checks. */
7790         if (recommended_table_type_name)
7791                 CHECK_NAME(recommended_table_type_name, EINVAL);
7792
7793         if (params->n_fields) {
7794                 enum rte_swx_table_match_type match_type;
7795
7796                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7797                 if (status)
7798                         return status;
7799
7800                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7801                 CHECK(type, EINVAL);
7802         } else {
7803                 type = NULL;
7804         }
7805
7806         /* Memory allocation. */
7807         t = calloc(1, sizeof(struct table));
7808         if (!t) {
7809                 status = -ENOMEM;
7810                 goto error;
7811         }
7812
7813         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7814         if (!t->fields) {
7815                 status = -ENOMEM;
7816                 goto error;
7817         }
7818
7819         t->actions = calloc(params->n_actions, sizeof(struct action *));
7820         if (!t->actions) {
7821                 status = -ENOMEM;
7822                 goto error;
7823         }
7824
7825         if (action_data_size_max) {
7826                 t->default_action_data = calloc(1, action_data_size_max);
7827                 if (!t->default_action_data) {
7828                         status = -ENOMEM;
7829                         goto error;
7830                 }
7831         }
7832
7833         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7834         if (!t->action_is_for_table_entries) {
7835                 status = -ENOMEM;
7836                 goto error;
7837         }
7838
7839         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7840         if (!t->action_is_for_default_entry) {
7841                 status = -ENOMEM;
7842                 goto error;
7843         }
7844
7845         /* Node initialization. */
7846         strcpy(t->name, name);
7847         if (args && args[0])
7848                 strcpy(t->args, args);
7849         t->type = type;
7850
7851         for (i = 0; i < params->n_fields; i++) {
7852                 struct rte_swx_match_field_params *field = &params->fields[i];
7853                 struct match_field *f = &t->fields[i];
7854
7855                 f->match_type = field->match_type;
7856                 f->field = header ?
7857                         header_field_parse(p, field->name, NULL) :
7858                         metadata_field_parse(p, field->name);
7859         }
7860         t->n_fields = params->n_fields;
7861         t->header = header;
7862
7863         for (i = 0; i < params->n_actions; i++) {
7864                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7865
7866                 if (params->action_is_for_table_entries)
7867                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7868                 if (params->action_is_for_default_entry)
7869                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7870
7871                 t->actions[i] = action_find(p, params->action_names[i]);
7872                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7873                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7874         }
7875         t->default_action = default_action;
7876         if (default_action->st) {
7877                 status = action_args_parse(default_action,
7878                                            params->default_action_args,
7879                                            t->default_action_data);
7880                 if (status)
7881                         goto error;
7882         }
7883
7884         t->n_actions = params->n_actions;
7885         t->default_action_is_const = params->default_action_is_const;
7886         t->action_data_size_max = action_data_size_max;
7887
7888         t->size = size;
7889         t->id = p->n_tables;
7890
7891         /* Node add to tailq. */
7892         TAILQ_INSERT_TAIL(&p->tables, t, node);
7893         p->n_tables++;
7894
7895         return 0;
7896
7897 error:
7898         if (!t)
7899                 return status;
7900
7901         free(t->action_is_for_default_entry);
7902         free(t->action_is_for_table_entries);
7903         free(t->default_action_data);
7904         free(t->actions);
7905         free(t->fields);
7906         free(t);
7907
7908         return status;
7909 }
7910
7911 static struct rte_swx_table_params *
7912 table_params_get(struct table *table)
7913 {
7914         struct rte_swx_table_params *params;
7915         struct field *first, *last;
7916         uint8_t *key_mask;
7917         uint32_t key_size, key_offset, action_data_size, i;
7918
7919         /* Memory allocation. */
7920         params = calloc(1, sizeof(struct rte_swx_table_params));
7921         if (!params)
7922                 return NULL;
7923
7924         /* Find first (smallest offset) and last (biggest offset) match fields. */
7925         first = table->fields[0].field;
7926         last = table->fields[0].field;
7927
7928         for (i = 0; i < table->n_fields; i++) {
7929                 struct field *f = table->fields[i].field;
7930
7931                 if (f->offset < first->offset)
7932                         first = f;
7933
7934                 if (f->offset > last->offset)
7935                         last = f;
7936         }
7937
7938         /* Key offset and size. */
7939         key_offset = first->offset / 8;
7940         key_size = (last->offset + last->n_bits - first->offset) / 8;
7941
7942         /* Memory allocation. */
7943         key_mask = calloc(1, key_size);
7944         if (!key_mask) {
7945                 free(params);
7946                 return NULL;
7947         }
7948
7949         /* Key mask. */
7950         for (i = 0; i < table->n_fields; i++) {
7951                 struct field *f = table->fields[i].field;
7952                 uint32_t start = (f->offset - first->offset) / 8;
7953                 size_t size = f->n_bits / 8;
7954
7955                 memset(&key_mask[start], 0xFF, size);
7956         }
7957
7958         /* Action data size. */
7959         action_data_size = 0;
7960         for (i = 0; i < table->n_actions; i++) {
7961                 struct action *action = table->actions[i];
7962                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7963
7964                 if (ads > action_data_size)
7965                         action_data_size = ads;
7966         }
7967
7968         /* Fill in. */
7969         params->match_type = table->type->match_type;
7970         params->key_size = key_size;
7971         params->key_offset = key_offset;
7972         params->key_mask0 = key_mask;
7973         params->action_data_size = action_data_size;
7974         params->n_keys_max = table->size;
7975
7976         return params;
7977 }
7978
7979 static void
7980 table_params_free(struct rte_swx_table_params *params)
7981 {
7982         if (!params)
7983                 return;
7984
7985         free(params->key_mask0);
7986         free(params);
7987 }
7988
7989 static int
7990 table_stub_lkp(void *table __rte_unused,
7991                void *mailbox __rte_unused,
7992                uint8_t **key __rte_unused,
7993                uint64_t *action_id __rte_unused,
7994                uint8_t **action_data __rte_unused,
7995                int *hit)
7996 {
7997         *hit = 0;
7998         return 1; /* DONE. */
7999 }
8000
8001 static int
8002 table_build(struct rte_swx_pipeline *p)
8003 {
8004         uint32_t i;
8005
8006         /* Per pipeline: table statistics. */
8007         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8008         CHECK(p->table_stats, ENOMEM);
8009
8010         for (i = 0; i < p->n_tables; i++) {
8011                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8012                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8013         }
8014
8015         /* Per thread: table runt-time. */
8016         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8017                 struct thread *t = &p->threads[i];
8018                 struct table *table;
8019
8020                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8021                 CHECK(t->tables, ENOMEM);
8022
8023                 TAILQ_FOREACH(table, &p->tables, node) {
8024                         struct table_runtime *r = &t->tables[table->id];
8025
8026                         if (table->type) {
8027                                 uint64_t size;
8028
8029                                 size = table->type->ops.mailbox_size_get();
8030
8031                                 /* r->func. */
8032                                 r->func = table->type->ops.lkp;
8033
8034                                 /* r->mailbox. */
8035                                 if (size) {
8036                                         r->mailbox = calloc(1, size);
8037                                         CHECK(r->mailbox, ENOMEM);
8038                                 }
8039
8040                                 /* r->key. */
8041                                 r->key = table->header ?
8042                                         &t->structs[table->header->struct_id] :
8043                                         &t->structs[p->metadata_struct_id];
8044                         } else {
8045                                 r->func = table_stub_lkp;
8046                         }
8047                 }
8048         }
8049
8050         return 0;
8051 }
8052
8053 static void
8054 table_build_free(struct rte_swx_pipeline *p)
8055 {
8056         uint32_t i;
8057
8058         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8059                 struct thread *t = &p->threads[i];
8060                 uint32_t j;
8061
8062                 if (!t->tables)
8063                         continue;
8064
8065                 for (j = 0; j < p->n_tables; j++) {
8066                         struct table_runtime *r = &t->tables[j];
8067
8068                         free(r->mailbox);
8069                 }
8070
8071                 free(t->tables);
8072                 t->tables = NULL;
8073         }
8074
8075         if (p->table_stats) {
8076                 for (i = 0; i < p->n_tables; i++)
8077                         free(p->table_stats[i].n_pkts_action);
8078
8079                 free(p->table_stats);
8080         }
8081 }
8082
8083 static void
8084 table_free(struct rte_swx_pipeline *p)
8085 {
8086         table_build_free(p);
8087
8088         /* Tables. */
8089         for ( ; ; ) {
8090                 struct table *elem;
8091
8092                 elem = TAILQ_FIRST(&p->tables);
8093                 if (!elem)
8094                         break;
8095
8096                 TAILQ_REMOVE(&p->tables, elem, node);
8097                 free(elem->fields);
8098                 free(elem->actions);
8099                 free(elem->default_action_data);
8100                 free(elem);
8101         }
8102
8103         /* Table types. */
8104         for ( ; ; ) {
8105                 struct table_type *elem;
8106
8107                 elem = TAILQ_FIRST(&p->table_types);
8108                 if (!elem)
8109                         break;
8110
8111                 TAILQ_REMOVE(&p->table_types, elem, node);
8112                 free(elem);
8113         }
8114 }
8115
8116 /*
8117  * Selector.
8118  */
8119 static struct selector *
8120 selector_find(struct rte_swx_pipeline *p, const char *name)
8121 {
8122         struct selector *s;
8123
8124         TAILQ_FOREACH(s, &p->selectors, node)
8125                 if (strcmp(s->name, name) == 0)
8126                         return s;
8127
8128         return NULL;
8129 }
8130
8131 static struct selector *
8132 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8133 {
8134         struct selector *s = NULL;
8135
8136         TAILQ_FOREACH(s, &p->selectors, node)
8137                 if (s->id == id)
8138                         return s;
8139
8140         return NULL;
8141 }
8142
8143 static int
8144 selector_fields_check(struct rte_swx_pipeline *p,
8145                       struct rte_swx_pipeline_selector_params *params,
8146                       struct header **header)
8147 {
8148         struct header *h0 = NULL;
8149         struct field *hf, *mf;
8150         uint32_t i;
8151
8152         /* Return if no selector fields. */
8153         if (!params->n_selector_fields || !params->selector_field_names)
8154                 return -EINVAL;
8155
8156         /* Check that all the selector fields either belong to the same header
8157          * or are all meta-data fields.
8158          */
8159         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8160         mf = metadata_field_parse(p, params->selector_field_names[0]);
8161         if (!hf && !mf)
8162                 return -EINVAL;
8163
8164         for (i = 1; i < params->n_selector_fields; i++)
8165                 if (h0) {
8166                         struct header *h;
8167
8168                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8169                         if (!hf || (h->id != h0->id))
8170                                 return -EINVAL;
8171                 } else {
8172                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8173                         if (!mf)
8174                                 return -EINVAL;
8175                 }
8176
8177         /* Check that there are no duplicated match fields. */
8178         for (i = 0; i < params->n_selector_fields; i++) {
8179                 const char *field_name = params->selector_field_names[i];
8180                 uint32_t j;
8181
8182                 for (j = i + 1; j < params->n_selector_fields; j++)
8183                         if (!strcmp(params->selector_field_names[j], field_name))
8184                                 return -EINVAL;
8185         }
8186
8187         /* Return. */
8188         if (header)
8189                 *header = h0;
8190
8191         return 0;
8192 }
8193
8194 int
8195 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8196                                  const char *name,
8197                                  struct rte_swx_pipeline_selector_params *params)
8198 {
8199         struct selector *s;
8200         struct header *selector_header = NULL;
8201         struct field *group_id_field, *member_id_field;
8202         uint32_t i;
8203         int status = 0;
8204
8205         CHECK(p, EINVAL);
8206
8207         CHECK_NAME(name, EINVAL);
8208         CHECK(!table_find(p, name), EEXIST);
8209         CHECK(!selector_find(p, name), EEXIST);
8210         CHECK(!learner_find(p, name), EEXIST);
8211
8212         CHECK(params, EINVAL);
8213
8214         CHECK_NAME(params->group_id_field_name, EINVAL);
8215         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8216         CHECK(group_id_field, EINVAL);
8217
8218         for (i = 0; i < params->n_selector_fields; i++) {
8219                 const char *field_name = params->selector_field_names[i];
8220
8221                 CHECK_NAME(field_name, EINVAL);
8222         }
8223         status = selector_fields_check(p, params, &selector_header);
8224         if (status)
8225                 return status;
8226
8227         CHECK_NAME(params->member_id_field_name, EINVAL);
8228         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8229         CHECK(member_id_field, EINVAL);
8230
8231         CHECK(params->n_groups_max, EINVAL);
8232
8233         CHECK(params->n_members_per_group_max, EINVAL);
8234
8235         /* Memory allocation. */
8236         s = calloc(1, sizeof(struct selector));
8237         if (!s) {
8238                 status = -ENOMEM;
8239                 goto error;
8240         }
8241
8242         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8243         if (!s->selector_fields) {
8244                 status = -ENOMEM;
8245                 goto error;
8246         }
8247
8248         /* Node initialization. */
8249         strcpy(s->name, name);
8250
8251         s->group_id_field = group_id_field;
8252
8253         for (i = 0; i < params->n_selector_fields; i++) {
8254                 const char *field_name = params->selector_field_names[i];
8255
8256                 s->selector_fields[i] = selector_header ?
8257                         header_field_parse(p, field_name, NULL) :
8258                         metadata_field_parse(p, field_name);
8259         }
8260
8261         s->n_selector_fields = params->n_selector_fields;
8262
8263         s->selector_header = selector_header;
8264
8265         s->member_id_field = member_id_field;
8266
8267         s->n_groups_max = params->n_groups_max;
8268
8269         s->n_members_per_group_max = params->n_members_per_group_max;
8270
8271         s->id = p->n_selectors;
8272
8273         /* Node add to tailq. */
8274         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8275         p->n_selectors++;
8276
8277         return 0;
8278
8279 error:
8280         if (!s)
8281                 return status;
8282
8283         free(s->selector_fields);
8284
8285         free(s);
8286
8287         return status;
8288 }
8289
8290 static void
8291 selector_params_free(struct rte_swx_table_selector_params *params)
8292 {
8293         if (!params)
8294                 return;
8295
8296         free(params->selector_mask);
8297
8298         free(params);
8299 }
8300
8301 static struct rte_swx_table_selector_params *
8302 selector_table_params_get(struct selector *s)
8303 {
8304         struct rte_swx_table_selector_params *params = NULL;
8305         struct field *first, *last;
8306         uint32_t i;
8307
8308         /* Memory allocation. */
8309         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8310         if (!params)
8311                 goto error;
8312
8313         /* Group ID. */
8314         params->group_id_offset = s->group_id_field->offset / 8;
8315
8316         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8317         first = s->selector_fields[0];
8318         last = s->selector_fields[0];
8319
8320         for (i = 0; i < s->n_selector_fields; i++) {
8321                 struct field *f = s->selector_fields[i];
8322
8323                 if (f->offset < first->offset)
8324                         first = f;
8325
8326                 if (f->offset > last->offset)
8327                         last = f;
8328         }
8329
8330         /* Selector offset and size. */
8331         params->selector_offset = first->offset / 8;
8332         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8333
8334         /* Memory allocation. */
8335         params->selector_mask = calloc(1, params->selector_size);
8336         if (!params->selector_mask)
8337                 goto error;
8338
8339         /* Selector mask. */
8340         for (i = 0; i < s->n_selector_fields; i++) {
8341                 struct field *f = s->selector_fields[i];
8342                 uint32_t start = (f->offset - first->offset) / 8;
8343                 size_t size = f->n_bits / 8;
8344
8345                 memset(&params->selector_mask[start], 0xFF, size);
8346         }
8347
8348         /* Member ID. */
8349         params->member_id_offset = s->member_id_field->offset / 8;
8350
8351         /* Maximum number of groups. */
8352         params->n_groups_max = s->n_groups_max;
8353
8354         /* Maximum number of members per group. */
8355         params->n_members_per_group_max = s->n_members_per_group_max;
8356
8357         return params;
8358
8359 error:
8360         selector_params_free(params);
8361         return NULL;
8362 }
8363
8364 static void
8365 selector_build_free(struct rte_swx_pipeline *p)
8366 {
8367         uint32_t i;
8368
8369         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8370                 struct thread *t = &p->threads[i];
8371                 uint32_t j;
8372
8373                 if (!t->selectors)
8374                         continue;
8375
8376                 for (j = 0; j < p->n_selectors; j++) {
8377                         struct selector_runtime *r = &t->selectors[j];
8378
8379                         free(r->mailbox);
8380                 }
8381
8382                 free(t->selectors);
8383                 t->selectors = NULL;
8384         }
8385
8386         free(p->selector_stats);
8387         p->selector_stats = NULL;
8388 }
8389
8390 static int
8391 selector_build(struct rte_swx_pipeline *p)
8392 {
8393         uint32_t i;
8394         int status = 0;
8395
8396         /* Per pipeline: selector statistics. */
8397         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8398         if (!p->selector_stats) {
8399                 status = -ENOMEM;
8400                 goto error;
8401         }
8402
8403         /* Per thread: selector run-time. */
8404         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8405                 struct thread *t = &p->threads[i];
8406                 struct selector *s;
8407
8408                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8409                 if (!t->selectors) {
8410                         status = -ENOMEM;
8411                         goto error;
8412                 }
8413
8414                 TAILQ_FOREACH(s, &p->selectors, node) {
8415                         struct selector_runtime *r = &t->selectors[s->id];
8416                         uint64_t size;
8417
8418                         /* r->mailbox. */
8419                         size = rte_swx_table_selector_mailbox_size_get();
8420                         if (size) {
8421                                 r->mailbox = calloc(1, size);
8422                                 if (!r->mailbox) {
8423                                         status = -ENOMEM;
8424                                         goto error;
8425                                 }
8426                         }
8427
8428                         /* r->group_id_buffer. */
8429                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8430
8431                         /* r->selector_buffer. */
8432                         r->selector_buffer = s->selector_header ?
8433                                 &t->structs[s->selector_header->struct_id] :
8434                                 &t->structs[p->metadata_struct_id];
8435
8436                         /* r->member_id_buffer. */
8437                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8438                 }
8439         }
8440
8441         return 0;
8442
8443 error:
8444         selector_build_free(p);
8445         return status;
8446 }
8447
8448 static void
8449 selector_free(struct rte_swx_pipeline *p)
8450 {
8451         selector_build_free(p);
8452
8453         /* Selector tables. */
8454         for ( ; ; ) {
8455                 struct selector *elem;
8456
8457                 elem = TAILQ_FIRST(&p->selectors);
8458                 if (!elem)
8459                         break;
8460
8461                 TAILQ_REMOVE(&p->selectors, elem, node);
8462                 free(elem->selector_fields);
8463                 free(elem);
8464         }
8465 }
8466
8467 /*
8468  * Learner table.
8469  */
8470 static struct learner *
8471 learner_find(struct rte_swx_pipeline *p, const char *name)
8472 {
8473         struct learner *l;
8474
8475         TAILQ_FOREACH(l, &p->learners, node)
8476                 if (!strcmp(l->name, name))
8477                         return l;
8478
8479         return NULL;
8480 }
8481
8482 static struct learner *
8483 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8484 {
8485         struct learner *l = NULL;
8486
8487         TAILQ_FOREACH(l, &p->learners, node)
8488                 if (l->id == id)
8489                         return l;
8490
8491         return NULL;
8492 }
8493
8494 static int
8495 learner_match_fields_check(struct rte_swx_pipeline *p,
8496                            struct rte_swx_pipeline_learner_params *params,
8497                            struct header **header)
8498 {
8499         struct header *h0 = NULL;
8500         struct field *hf, *mf;
8501         uint32_t i;
8502
8503         /* Return if no match fields. */
8504         if (!params->n_fields || !params->field_names)
8505                 return -EINVAL;
8506
8507         /* Check that all the match fields either belong to the same header
8508          * or are all meta-data fields.
8509          */
8510         hf = header_field_parse(p, params->field_names[0], &h0);
8511         mf = metadata_field_parse(p, params->field_names[0]);
8512         if (!hf && !mf)
8513                 return -EINVAL;
8514
8515         for (i = 1; i < params->n_fields; i++)
8516                 if (h0) {
8517                         struct header *h;
8518
8519                         hf = header_field_parse(p, params->field_names[i], &h);
8520                         if (!hf || (h->id != h0->id))
8521                                 return -EINVAL;
8522                 } else {
8523                         mf = metadata_field_parse(p, params->field_names[i]);
8524                         if (!mf)
8525                                 return -EINVAL;
8526                 }
8527
8528         /* Check that there are no duplicated match fields. */
8529         for (i = 0; i < params->n_fields; i++) {
8530                 const char *field_name = params->field_names[i];
8531                 uint32_t j;
8532
8533                 for (j = i + 1; j < params->n_fields; j++)
8534                         if (!strcmp(params->field_names[j], field_name))
8535                                 return -EINVAL;
8536         }
8537
8538         /* Return. */
8539         if (header)
8540                 *header = h0;
8541
8542         return 0;
8543 }
8544
8545 static int
8546 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8547 {
8548         struct struct_type *mst = p->metadata_st, *ast = a->st;
8549         struct field *mf, *af;
8550         uint32_t mf_pos, i;
8551
8552         if (!ast) {
8553                 if (mf_name)
8554                         return -EINVAL;
8555
8556                 return 0;
8557         }
8558
8559         /* Check that mf_name is the name of a valid meta-data field. */
8560         CHECK_NAME(mf_name, EINVAL);
8561         mf = metadata_field_parse(p, mf_name);
8562         CHECK(mf, EINVAL);
8563
8564         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8565          * all the action arguments.
8566          */
8567         mf_pos = mf - mst->fields;
8568         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8569
8570         /* Check that the size of each of the identified meta-data fields matches exactly the size
8571          * of the corresponding action argument.
8572          */
8573         for (i = 0; i < ast->n_fields; i++) {
8574                 mf = &mst->fields[mf_pos + i];
8575                 af = &ast->fields[i];
8576
8577                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8578         }
8579
8580         return 0;
8581 }
8582
8583 static int
8584 learner_action_learning_check(struct rte_swx_pipeline *p,
8585                               struct action *action,
8586                               const char **action_names,
8587                               uint32_t n_actions)
8588 {
8589         uint32_t i;
8590
8591         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8592          * the action passed as argument to the "learn" instruction) is also enabled for the
8593          * current learner table.
8594          */
8595         for (i = 0; i < action->n_instructions; i++) {
8596                 struct instruction *instr = &action->instructions[i];
8597                 uint32_t found = 0, j;
8598
8599                 if (instr->type != INSTR_LEARNER_LEARN)
8600                         continue;
8601
8602                 for (j = 0; j < n_actions; j++) {
8603                         struct action *a;
8604
8605                         a = action_find(p, action_names[j]);
8606                         if (!a)
8607                                 return -EINVAL;
8608
8609                         if (a->id == instr->learn.action_id)
8610                                 found = 1;
8611                 }
8612
8613                 if (!found)
8614                         return -EINVAL;
8615         }
8616
8617         return 0;
8618 }
8619
8620 int
8621 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8622                               const char *name,
8623                               struct rte_swx_pipeline_learner_params *params,
8624                               uint32_t size,
8625                               uint32_t *timeout,
8626                               uint32_t n_timeouts)
8627 {
8628         struct learner *l = NULL;
8629         struct action *default_action;
8630         struct header *header = NULL;
8631         uint32_t action_data_size_max = 0, i;
8632         int status = 0;
8633
8634         CHECK(p, EINVAL);
8635
8636         CHECK_NAME(name, EINVAL);
8637         CHECK(!table_find(p, name), EEXIST);
8638         CHECK(!selector_find(p, name), EEXIST);
8639         CHECK(!learner_find(p, name), EEXIST);
8640
8641         CHECK(params, EINVAL);
8642
8643         /* Match checks. */
8644         status = learner_match_fields_check(p, params, &header);
8645         if (status)
8646                 return status;
8647
8648         /* Action checks. */
8649         CHECK(params->n_actions, EINVAL);
8650         CHECK(params->action_names, EINVAL);
8651         for (i = 0; i < params->n_actions; i++) {
8652                 const char *action_name = params->action_names[i];
8653                 struct action *a;
8654                 uint32_t action_data_size;
8655                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8656
8657                 CHECK_NAME(action_name, EINVAL);
8658
8659                 a = action_find(p, action_name);
8660                 CHECK(a, EINVAL);
8661
8662                 status = learner_action_learning_check(p,
8663                                                        a,
8664                                                        params->action_names,
8665                                                        params->n_actions);
8666                 if (status)
8667                         return status;
8668
8669                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8670                 if (action_data_size > action_data_size_max)
8671                         action_data_size_max = action_data_size;
8672
8673                 if (params->action_is_for_table_entries)
8674                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8675                 if (params->action_is_for_default_entry)
8676                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8677                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8678         }
8679
8680         CHECK_NAME(params->default_action_name, EINVAL);
8681         for (i = 0; i < p->n_actions; i++)
8682                 if (!strcmp(params->action_names[i],
8683                             params->default_action_name))
8684                         break;
8685         CHECK(i < params->n_actions, EINVAL);
8686         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8687               EINVAL);
8688
8689         default_action = action_find(p, params->default_action_name);
8690         CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
8691               EINVAL);
8692
8693         /* Any other checks. */
8694         CHECK(size, EINVAL);
8695         CHECK(timeout, EINVAL);
8696         CHECK(n_timeouts && (n_timeouts < RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX), EINVAL);
8697
8698         /* Memory allocation. */
8699         l = calloc(1, sizeof(struct learner));
8700         if (!l) {
8701                 status = -ENOMEM;
8702                 goto error;
8703         }
8704
8705         l->fields = calloc(params->n_fields, sizeof(struct field *));
8706         if (!l->fields) {
8707                 status = -ENOMEM;
8708                 goto error;
8709         }
8710
8711         l->actions = calloc(params->n_actions, sizeof(struct action *));
8712         if (!l->actions) {
8713                 status = -ENOMEM;
8714                 goto error;
8715         }
8716
8717         if (action_data_size_max) {
8718                 l->default_action_data = calloc(1, action_data_size_max);
8719                 if (!l->default_action_data) {
8720                         status = -ENOMEM;
8721                         goto error;
8722                 }
8723         }
8724
8725         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8726         if (!l->action_is_for_table_entries) {
8727                 status = -ENOMEM;
8728                 goto error;
8729         }
8730
8731         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8732         if (!l->action_is_for_default_entry) {
8733                 status = -ENOMEM;
8734                 goto error;
8735         }
8736
8737         /* Node initialization. */
8738         strcpy(l->name, name);
8739
8740         for (i = 0; i < params->n_fields; i++) {
8741                 const char *field_name = params->field_names[i];
8742
8743                 l->fields[i] = header ?
8744                         header_field_parse(p, field_name, NULL) :
8745                         metadata_field_parse(p, field_name);
8746         }
8747
8748         l->n_fields = params->n_fields;
8749
8750         l->header = header;
8751
8752         for (i = 0; i < params->n_actions; i++) {
8753                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8754
8755                 if (params->action_is_for_table_entries)
8756                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8757                 if (params->action_is_for_default_entry)
8758                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8759
8760                 l->actions[i] = action_find(p, params->action_names[i]);
8761                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8762                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8763         }
8764
8765         l->default_action = default_action;
8766
8767         if (default_action->st) {
8768                 status = action_args_parse(default_action,
8769                                            params->default_action_args,
8770                                            l->default_action_data);
8771                 if (status)
8772                         goto error;
8773         }
8774
8775         l->n_actions = params->n_actions;
8776
8777         l->default_action_is_const = params->default_action_is_const;
8778
8779         l->action_data_size_max = action_data_size_max;
8780
8781         l->size = size;
8782
8783         for (i = 0; i < n_timeouts; i++)
8784                 l->timeout[i] = timeout[i];
8785
8786         l->n_timeouts = n_timeouts;
8787
8788         l->id = p->n_learners;
8789
8790         /* Node add to tailq. */
8791         TAILQ_INSERT_TAIL(&p->learners, l, node);
8792         p->n_learners++;
8793
8794         return 0;
8795
8796 error:
8797         if (!l)
8798                 return status;
8799
8800         free(l->action_is_for_default_entry);
8801         free(l->action_is_for_table_entries);
8802         free(l->default_action_data);
8803         free(l->actions);
8804         free(l->fields);
8805         free(l);
8806
8807         return status;
8808 }
8809
8810 static void
8811 learner_params_free(struct rte_swx_table_learner_params *params)
8812 {
8813         if (!params)
8814                 return;
8815
8816         free(params->key_mask0);
8817
8818         free(params->key_timeout);
8819
8820         free(params);
8821 }
8822
8823 static struct rte_swx_table_learner_params *
8824 learner_params_get(struct learner *l)
8825 {
8826         struct rte_swx_table_learner_params *params = NULL;
8827         struct field *first, *last;
8828         uint32_t i;
8829
8830         /* Memory allocation. */
8831         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8832         if (!params)
8833                 goto error;
8834
8835         /* Find first (smallest offset) and last (biggest offset) match fields. */
8836         first = l->fields[0];
8837         last = l->fields[0];
8838
8839         for (i = 0; i < l->n_fields; i++) {
8840                 struct field *f = l->fields[i];
8841
8842                 if (f->offset < first->offset)
8843                         first = f;
8844
8845                 if (f->offset > last->offset)
8846                         last = f;
8847         }
8848
8849         /* Key offset and size. */
8850         params->key_offset = first->offset / 8;
8851         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8852
8853         /* Memory allocation. */
8854         params->key_mask0 = calloc(1, params->key_size);
8855         if (!params->key_mask0)
8856                 goto error;
8857
8858         /* Key mask. */
8859         for (i = 0; i < l->n_fields; i++) {
8860                 struct field *f = l->fields[i];
8861                 uint32_t start = (f->offset - first->offset) / 8;
8862                 size_t size = f->n_bits / 8;
8863
8864                 memset(&params->key_mask0[start], 0xFF, size);
8865         }
8866
8867         /* Action data size. */
8868         params->action_data_size = l->action_data_size_max;
8869
8870         /* Maximum number of keys. */
8871         params->n_keys_max = l->size;
8872
8873         /* Memory allocation. */
8874         params->key_timeout = calloc(l->n_timeouts, sizeof(uint32_t));
8875         if (!params->key_timeout)
8876                 goto error;
8877
8878         /* Timeout. */
8879         for (i = 0; i < l->n_timeouts; i++)
8880                 params->key_timeout[i] = l->timeout[i];
8881
8882         params->n_key_timeouts = l->n_timeouts;
8883
8884         return params;
8885
8886 error:
8887         learner_params_free(params);
8888         return NULL;
8889 }
8890
8891 static void
8892 learner_build_free(struct rte_swx_pipeline *p)
8893 {
8894         uint32_t i;
8895
8896         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8897                 struct thread *t = &p->threads[i];
8898                 uint32_t j;
8899
8900                 if (!t->learners)
8901                         continue;
8902
8903                 for (j = 0; j < p->n_learners; j++) {
8904                         struct learner_runtime *r = &t->learners[j];
8905
8906                         free(r->mailbox);
8907                 }
8908
8909                 free(t->learners);
8910                 t->learners = NULL;
8911         }
8912
8913         if (p->learner_stats) {
8914                 for (i = 0; i < p->n_learners; i++)
8915                         free(p->learner_stats[i].n_pkts_action);
8916
8917                 free(p->learner_stats);
8918         }
8919 }
8920
8921 static int
8922 learner_build(struct rte_swx_pipeline *p)
8923 {
8924         uint32_t i;
8925         int status = 0;
8926
8927         /* Per pipeline: learner statistics. */
8928         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8929         CHECK(p->learner_stats, ENOMEM);
8930
8931         for (i = 0; i < p->n_learners; i++) {
8932                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8933                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8934         }
8935
8936         /* Per thread: learner run-time. */
8937         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8938                 struct thread *t = &p->threads[i];
8939                 struct learner *l;
8940
8941                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8942                 if (!t->learners) {
8943                         status = -ENOMEM;
8944                         goto error;
8945                 }
8946
8947                 TAILQ_FOREACH(l, &p->learners, node) {
8948                         struct learner_runtime *r = &t->learners[l->id];
8949                         uint64_t size;
8950
8951                         /* r->mailbox. */
8952                         size = rte_swx_table_learner_mailbox_size_get();
8953                         if (size) {
8954                                 r->mailbox = calloc(1, size);
8955                                 if (!r->mailbox) {
8956                                         status = -ENOMEM;
8957                                         goto error;
8958                                 }
8959                         }
8960
8961                         /* r->key. */
8962                         r->key = l->header ?
8963                                 &t->structs[l->header->struct_id] :
8964                                 &t->structs[p->metadata_struct_id];
8965                 }
8966         }
8967
8968         return 0;
8969
8970 error:
8971         learner_build_free(p);
8972         return status;
8973 }
8974
8975 static void
8976 learner_free(struct rte_swx_pipeline *p)
8977 {
8978         learner_build_free(p);
8979
8980         /* Learner tables. */
8981         for ( ; ; ) {
8982                 struct learner *l;
8983
8984                 l = TAILQ_FIRST(&p->learners);
8985                 if (!l)
8986                         break;
8987
8988                 TAILQ_REMOVE(&p->learners, l, node);
8989                 free(l->fields);
8990                 free(l->actions);
8991                 free(l->default_action_data);
8992                 free(l);
8993         }
8994 }
8995
8996 /*
8997  * Table state.
8998  */
8999 static int
9000 table_state_build(struct rte_swx_pipeline *p)
9001 {
9002         struct table *table;
9003         struct selector *s;
9004         struct learner *l;
9005
9006         p->table_state = calloc(p->n_tables + p->n_selectors + p->n_learners,
9007                                 sizeof(struct rte_swx_table_state));
9008         CHECK(p->table_state, ENOMEM);
9009
9010         TAILQ_FOREACH(table, &p->tables, node) {
9011                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9012
9013                 if (table->type) {
9014                         struct rte_swx_table_params *params;
9015
9016                         /* ts->obj. */
9017                         params = table_params_get(table);
9018                         CHECK(params, ENOMEM);
9019
9020                         ts->obj = table->type->ops.create(params,
9021                                 NULL,
9022                                 table->args,
9023                                 p->numa_node);
9024
9025                         table_params_free(params);
9026                         CHECK(ts->obj, ENODEV);
9027                 }
9028
9029                 /* ts->default_action_data. */
9030                 if (table->action_data_size_max) {
9031                         ts->default_action_data =
9032                                 malloc(table->action_data_size_max);
9033                         CHECK(ts->default_action_data, ENOMEM);
9034
9035                         memcpy(ts->default_action_data,
9036                                table->default_action_data,
9037                                table->action_data_size_max);
9038                 }
9039
9040                 /* ts->default_action_id. */
9041                 ts->default_action_id = table->default_action->id;
9042         }
9043
9044         TAILQ_FOREACH(s, &p->selectors, node) {
9045                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9046                 struct rte_swx_table_selector_params *params;
9047
9048                 /* ts->obj. */
9049                 params = selector_table_params_get(s);
9050                 CHECK(params, ENOMEM);
9051
9052                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9053
9054                 selector_params_free(params);
9055                 CHECK(ts->obj, ENODEV);
9056         }
9057
9058         TAILQ_FOREACH(l, &p->learners, node) {
9059                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9060                         p->n_selectors + l->id];
9061                 struct rte_swx_table_learner_params *params;
9062
9063                 /* ts->obj. */
9064                 params = learner_params_get(l);
9065                 CHECK(params, ENOMEM);
9066
9067                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9068                 learner_params_free(params);
9069                 CHECK(ts->obj, ENODEV);
9070
9071                 /* ts->default_action_data. */
9072                 if (l->action_data_size_max) {
9073                         ts->default_action_data = malloc(l->action_data_size_max);
9074                         CHECK(ts->default_action_data, ENOMEM);
9075
9076                         memcpy(ts->default_action_data,
9077                                l->default_action_data,
9078                                l->action_data_size_max);
9079                 }
9080
9081                 /* ts->default_action_id. */
9082                 ts->default_action_id = l->default_action->id;
9083         }
9084
9085         return 0;
9086 }
9087
9088 static void
9089 table_state_build_free(struct rte_swx_pipeline *p)
9090 {
9091         uint32_t i;
9092
9093         if (!p->table_state)
9094                 return;
9095
9096         for (i = 0; i < p->n_tables; i++) {
9097                 struct rte_swx_table_state *ts = &p->table_state[i];
9098                 struct table *table = table_find_by_id(p, i);
9099
9100                 /* ts->obj. */
9101                 if (table->type && ts->obj)
9102                         table->type->ops.free(ts->obj);
9103
9104                 /* ts->default_action_data. */
9105                 free(ts->default_action_data);
9106         }
9107
9108         for (i = 0; i < p->n_selectors; i++) {
9109                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9110
9111                 /* ts->obj. */
9112                 if (ts->obj)
9113                         rte_swx_table_selector_free(ts->obj);
9114         }
9115
9116         for (i = 0; i < p->n_learners; i++) {
9117                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9118
9119                 /* ts->obj. */
9120                 if (ts->obj)
9121                         rte_swx_table_learner_free(ts->obj);
9122
9123                 /* ts->default_action_data. */
9124                 free(ts->default_action_data);
9125         }
9126
9127         free(p->table_state);
9128         p->table_state = NULL;
9129 }
9130
9131 static void
9132 table_state_free(struct rte_swx_pipeline *p)
9133 {
9134         table_state_build_free(p);
9135 }
9136
9137 /*
9138  * Register array.
9139  */
9140 static struct regarray *
9141 regarray_find(struct rte_swx_pipeline *p, const char *name)
9142 {
9143         struct regarray *elem;
9144
9145         TAILQ_FOREACH(elem, &p->regarrays, node)
9146                 if (!strcmp(elem->name, name))
9147                         return elem;
9148
9149         return NULL;
9150 }
9151
9152 static struct regarray *
9153 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9154 {
9155         struct regarray *elem = NULL;
9156
9157         TAILQ_FOREACH(elem, &p->regarrays, node)
9158                 if (elem->id == id)
9159                         return elem;
9160
9161         return NULL;
9162 }
9163
9164 int
9165 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9166                               const char *name,
9167                               uint32_t size,
9168                               uint64_t init_val)
9169 {
9170         struct regarray *r;
9171
9172         CHECK(p, EINVAL);
9173
9174         CHECK_NAME(name, EINVAL);
9175         CHECK(!regarray_find(p, name), EEXIST);
9176
9177         CHECK(size, EINVAL);
9178         size = rte_align32pow2(size);
9179
9180         /* Memory allocation. */
9181         r = calloc(1, sizeof(struct regarray));
9182         CHECK(r, ENOMEM);
9183
9184         /* Node initialization. */
9185         strcpy(r->name, name);
9186         r->init_val = init_val;
9187         r->size = size;
9188         r->id = p->n_regarrays;
9189
9190         /* Node add to tailq. */
9191         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9192         p->n_regarrays++;
9193
9194         return 0;
9195 }
9196
9197 static int
9198 regarray_build(struct rte_swx_pipeline *p)
9199 {
9200         struct regarray *regarray;
9201
9202         if (!p->n_regarrays)
9203                 return 0;
9204
9205         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9206         CHECK(p->regarray_runtime, ENOMEM);
9207
9208         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9209                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9210                 uint32_t i;
9211
9212                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9213                                          RTE_CACHE_LINE_SIZE,
9214                                          p->numa_node);
9215                 CHECK(r->regarray, ENOMEM);
9216
9217                 if (regarray->init_val)
9218                         for (i = 0; i < regarray->size; i++)
9219                                 r->regarray[i] = regarray->init_val;
9220
9221                 r->size_mask = regarray->size - 1;
9222         }
9223
9224         return 0;
9225 }
9226
9227 static void
9228 regarray_build_free(struct rte_swx_pipeline *p)
9229 {
9230         uint32_t i;
9231
9232         if (!p->regarray_runtime)
9233                 return;
9234
9235         for (i = 0; i < p->n_regarrays; i++) {
9236                 struct regarray *regarray = regarray_find_by_id(p, i);
9237                 struct regarray_runtime *r = &p->regarray_runtime[i];
9238
9239                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9240         }
9241
9242         free(p->regarray_runtime);
9243         p->regarray_runtime = NULL;
9244 }
9245
9246 static void
9247 regarray_free(struct rte_swx_pipeline *p)
9248 {
9249         regarray_build_free(p);
9250
9251         for ( ; ; ) {
9252                 struct regarray *elem;
9253
9254                 elem = TAILQ_FIRST(&p->regarrays);
9255                 if (!elem)
9256                         break;
9257
9258                 TAILQ_REMOVE(&p->regarrays, elem, node);
9259                 free(elem);
9260         }
9261 }
9262
9263 /*
9264  * Meter array.
9265  */
9266 static struct meter_profile *
9267 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9268 {
9269         struct meter_profile *elem;
9270
9271         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9272                 if (!strcmp(elem->name, name))
9273                         return elem;
9274
9275         return NULL;
9276 }
9277
9278 static struct metarray *
9279 metarray_find(struct rte_swx_pipeline *p, const char *name)
9280 {
9281         struct metarray *elem;
9282
9283         TAILQ_FOREACH(elem, &p->metarrays, node)
9284                 if (!strcmp(elem->name, name))
9285                         return elem;
9286
9287         return NULL;
9288 }
9289
9290 static struct metarray *
9291 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9292 {
9293         struct metarray *elem = NULL;
9294
9295         TAILQ_FOREACH(elem, &p->metarrays, node)
9296                 if (elem->id == id)
9297                         return elem;
9298
9299         return NULL;
9300 }
9301
9302 int
9303 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9304                                  const char *name,
9305                                  uint32_t size)
9306 {
9307         struct metarray *m;
9308
9309         CHECK(p, EINVAL);
9310
9311         CHECK_NAME(name, EINVAL);
9312         CHECK(!metarray_find(p, name), EEXIST);
9313
9314         CHECK(size, EINVAL);
9315         size = rte_align32pow2(size);
9316
9317         /* Memory allocation. */
9318         m = calloc(1, sizeof(struct metarray));
9319         CHECK(m, ENOMEM);
9320
9321         /* Node initialization. */
9322         strcpy(m->name, name);
9323         m->size = size;
9324         m->id = p->n_metarrays;
9325
9326         /* Node add to tailq. */
9327         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9328         p->n_metarrays++;
9329
9330         return 0;
9331 }
9332
9333 struct meter_profile meter_profile_default = {
9334         .node = {0},
9335         .name = "",
9336         .params = {0},
9337
9338         .profile = {
9339                 .cbs = 10000,
9340                 .pbs = 10000,
9341                 .cir_period = 1,
9342                 .cir_bytes_per_period = 1,
9343                 .pir_period = 1,
9344                 .pir_bytes_per_period = 1,
9345         },
9346
9347         .n_users = 0,
9348 };
9349
9350 static void
9351 meter_init(struct meter *m)
9352 {
9353         memset(m, 0, sizeof(struct meter));
9354         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9355         m->profile = &meter_profile_default;
9356         m->color_mask = RTE_COLOR_GREEN;
9357
9358         meter_profile_default.n_users++;
9359 }
9360
9361 static int
9362 metarray_build(struct rte_swx_pipeline *p)
9363 {
9364         struct metarray *m;
9365
9366         if (!p->n_metarrays)
9367                 return 0;
9368
9369         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9370         CHECK(p->metarray_runtime, ENOMEM);
9371
9372         TAILQ_FOREACH(m, &p->metarrays, node) {
9373                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9374                 uint32_t i;
9375
9376                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9377                                          RTE_CACHE_LINE_SIZE,
9378                                          p->numa_node);
9379                 CHECK(r->metarray, ENOMEM);
9380
9381                 for (i = 0; i < m->size; i++)
9382                         meter_init(&r->metarray[i]);
9383
9384                 r->size_mask = m->size - 1;
9385         }
9386
9387         return 0;
9388 }
9389
9390 static void
9391 metarray_build_free(struct rte_swx_pipeline *p)
9392 {
9393         uint32_t i;
9394
9395         if (!p->metarray_runtime)
9396                 return;
9397
9398         for (i = 0; i < p->n_metarrays; i++) {
9399                 struct metarray *m = metarray_find_by_id(p, i);
9400                 struct metarray_runtime *r = &p->metarray_runtime[i];
9401
9402                 env_free(r->metarray, m->size * sizeof(struct meter));
9403         }
9404
9405         free(p->metarray_runtime);
9406         p->metarray_runtime = NULL;
9407 }
9408
9409 static void
9410 metarray_free(struct rte_swx_pipeline *p)
9411 {
9412         metarray_build_free(p);
9413
9414         /* Meter arrays. */
9415         for ( ; ; ) {
9416                 struct metarray *elem;
9417
9418                 elem = TAILQ_FIRST(&p->metarrays);
9419                 if (!elem)
9420                         break;
9421
9422                 TAILQ_REMOVE(&p->metarrays, elem, node);
9423                 free(elem);
9424         }
9425
9426         /* Meter profiles. */
9427         for ( ; ; ) {
9428                 struct meter_profile *elem;
9429
9430                 elem = TAILQ_FIRST(&p->meter_profiles);
9431                 if (!elem)
9432                         break;
9433
9434                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9435                 free(elem);
9436         }
9437 }
9438
9439 /*
9440  * Pipeline.
9441  */
9442 void
9443 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9444 {
9445         void *lib;
9446
9447         if (!p)
9448                 return;
9449
9450         lib = p->lib;
9451
9452         free(p->instruction_data);
9453         free(p->instructions);
9454
9455         metarray_free(p);
9456         regarray_free(p);
9457         table_state_free(p);
9458         learner_free(p);
9459         selector_free(p);
9460         table_free(p);
9461         action_free(p);
9462         instruction_table_free(p);
9463         metadata_free(p);
9464         header_free(p);
9465         extern_func_free(p);
9466         extern_obj_free(p);
9467         mirroring_free(p);
9468         port_out_free(p);
9469         port_in_free(p);
9470         struct_free(p);
9471
9472         free(p);
9473
9474         if (lib)
9475                 dlclose(lib);
9476 }
9477
9478 static int
9479 port_in_types_register(struct rte_swx_pipeline *p)
9480 {
9481         int status;
9482
9483         status = rte_swx_pipeline_port_in_type_register(p,
9484                 "ethdev",
9485                 &rte_swx_port_ethdev_reader_ops);
9486         if (status)
9487                 return status;
9488
9489         status = rte_swx_pipeline_port_in_type_register(p,
9490                 "ring",
9491                 &rte_swx_port_ring_reader_ops);
9492         if (status)
9493                 return status;
9494
9495 #ifdef RTE_PORT_PCAP
9496         status = rte_swx_pipeline_port_in_type_register(p,
9497                 "source",
9498                 &rte_swx_port_source_ops);
9499         if (status)
9500                 return status;
9501 #endif
9502
9503         status = rte_swx_pipeline_port_in_type_register(p,
9504                 "fd",
9505                 &rte_swx_port_fd_reader_ops);
9506         if (status)
9507                 return status;
9508
9509         return 0;
9510 }
9511
9512 static int
9513 port_out_types_register(struct rte_swx_pipeline *p)
9514 {
9515         int status;
9516
9517         status = rte_swx_pipeline_port_out_type_register(p,
9518                 "ethdev",
9519                 &rte_swx_port_ethdev_writer_ops);
9520         if (status)
9521                 return status;
9522
9523         status = rte_swx_pipeline_port_out_type_register(p,
9524                 "ring",
9525                 &rte_swx_port_ring_writer_ops);
9526         if (status)
9527                 return status;
9528
9529         status = rte_swx_pipeline_port_out_type_register(p,
9530                 "sink",
9531                 &rte_swx_port_sink_ops);
9532         if (status)
9533                 return status;
9534
9535         status = rte_swx_pipeline_port_out_type_register(p,
9536                 "fd",
9537                 &rte_swx_port_fd_writer_ops);
9538         if (status)
9539                 return status;
9540
9541         return 0;
9542 }
9543
9544 static int
9545 table_types_register(struct rte_swx_pipeline *p)
9546 {
9547         int status;
9548
9549         status = rte_swx_pipeline_table_type_register(p,
9550                 "exact",
9551                 RTE_SWX_TABLE_MATCH_EXACT,
9552                 &rte_swx_table_exact_match_ops);
9553         if (status)
9554                 return status;
9555
9556         status = rte_swx_pipeline_table_type_register(p,
9557                 "wildcard",
9558                 RTE_SWX_TABLE_MATCH_WILDCARD,
9559                 &rte_swx_table_wildcard_match_ops);
9560         if (status)
9561                 return status;
9562
9563         return 0;
9564 }
9565
9566 int
9567 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9568 {
9569         struct rte_swx_pipeline *pipeline = NULL;
9570         int status = 0;
9571
9572         /* Check input parameters. */
9573         CHECK(p, EINVAL);
9574
9575         /* Memory allocation. */
9576         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9577         if (!pipeline) {
9578                 status = -ENOMEM;
9579                 goto error;
9580         }
9581
9582         /* Initialization. */
9583         TAILQ_INIT(&pipeline->struct_types);
9584         TAILQ_INIT(&pipeline->port_in_types);
9585         TAILQ_INIT(&pipeline->ports_in);
9586         TAILQ_INIT(&pipeline->port_out_types);
9587         TAILQ_INIT(&pipeline->ports_out);
9588         TAILQ_INIT(&pipeline->extern_types);
9589         TAILQ_INIT(&pipeline->extern_objs);
9590         TAILQ_INIT(&pipeline->extern_funcs);
9591         TAILQ_INIT(&pipeline->headers);
9592         TAILQ_INIT(&pipeline->actions);
9593         TAILQ_INIT(&pipeline->table_types);
9594         TAILQ_INIT(&pipeline->tables);
9595         TAILQ_INIT(&pipeline->selectors);
9596         TAILQ_INIT(&pipeline->learners);
9597         TAILQ_INIT(&pipeline->regarrays);
9598         TAILQ_INIT(&pipeline->meter_profiles);
9599         TAILQ_INIT(&pipeline->metarrays);
9600
9601         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9602         pipeline->numa_node = numa_node;
9603
9604         status = port_in_types_register(pipeline);
9605         if (status)
9606                 goto error;
9607
9608         status = port_out_types_register(pipeline);
9609         if (status)
9610                 goto error;
9611
9612         status = table_types_register(pipeline);
9613         if (status)
9614                 goto error;
9615
9616         *p = pipeline;
9617         return 0;
9618
9619 error:
9620         rte_swx_pipeline_free(pipeline);
9621         return status;
9622 }
9623
9624 int
9625 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9626                                      const char **instructions,
9627                                      uint32_t n_instructions)
9628 {
9629         int err;
9630         uint32_t i;
9631
9632         err = instruction_config(p, NULL, instructions, n_instructions);
9633         if (err)
9634                 return err;
9635
9636         /* Thread instruction pointer reset. */
9637         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9638                 struct thread *t = &p->threads[i];
9639
9640                 thread_ip_reset(p, t);
9641         }
9642
9643         return 0;
9644 }
9645
9646 static int
9647 pipeline_compile(struct rte_swx_pipeline *p);
9648
9649 int
9650 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9651 {
9652         struct rte_swx_port_sink_params drop_port_params = {
9653                 .file_name = NULL,
9654         };
9655         int status;
9656
9657         CHECK(p, EINVAL);
9658         CHECK(p->build_done == 0, EEXIST);
9659
9660         status = port_in_build(p);
9661         if (status)
9662                 goto error;
9663
9664         /* Drop port. */
9665         status = rte_swx_pipeline_port_out_config(p,
9666                                                   p->n_ports_out,
9667                                                   "sink",
9668                                                   &drop_port_params);
9669         if (status)
9670                 goto error;
9671
9672         status = port_out_build(p);
9673         if (status)
9674                 goto error;
9675
9676         status = mirroring_build(p);
9677         if (status)
9678                 goto error;
9679
9680         status = struct_build(p);
9681         if (status)
9682                 goto error;
9683
9684         status = extern_obj_build(p);
9685         if (status)
9686                 goto error;
9687
9688         status = extern_func_build(p);
9689         if (status)
9690                 goto error;
9691
9692         status = header_build(p);
9693         if (status)
9694                 goto error;
9695
9696         status = metadata_build(p);
9697         if (status)
9698                 goto error;
9699
9700         status = instruction_table_build(p);
9701         if (status)
9702                 goto error;
9703
9704         status = action_build(p);
9705         if (status)
9706                 goto error;
9707
9708         status = table_build(p);
9709         if (status)
9710                 goto error;
9711
9712         status = selector_build(p);
9713         if (status)
9714                 goto error;
9715
9716         status = learner_build(p);
9717         if (status)
9718                 goto error;
9719
9720         status = table_state_build(p);
9721         if (status)
9722                 goto error;
9723
9724         status = regarray_build(p);
9725         if (status)
9726                 goto error;
9727
9728         status = metarray_build(p);
9729         if (status)
9730                 goto error;
9731
9732         p->build_done = 1;
9733
9734         pipeline_compile(p);
9735
9736         return 0;
9737
9738 error:
9739         metarray_build_free(p);
9740         regarray_build_free(p);
9741         table_state_build_free(p);
9742         learner_build_free(p);
9743         selector_build_free(p);
9744         table_build_free(p);
9745         action_build_free(p);
9746         instruction_table_build_free(p);
9747         metadata_build_free(p);
9748         header_build_free(p);
9749         extern_func_build_free(p);
9750         extern_obj_build_free(p);
9751         mirroring_build_free(p);
9752         port_out_build_free(p);
9753         port_in_build_free(p);
9754         struct_build_free(p);
9755
9756         return status;
9757 }
9758
9759 void
9760 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9761 {
9762         uint32_t i;
9763
9764         for (i = 0; i < n_instructions; i++)
9765                 instr_exec(p);
9766 }
9767
9768 void
9769 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9770 {
9771         uint32_t i;
9772
9773         for (i = 0; i < p->n_ports_out; i++) {
9774                 struct port_out_runtime *port = &p->out[i];
9775
9776                 if (port->flush)
9777                         port->flush(port->obj);
9778         }
9779 }
9780
9781 /*
9782  * Control.
9783  */
9784 int
9785 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9786                               struct rte_swx_ctl_pipeline_info *pipeline)
9787 {
9788         struct action *action;
9789         struct table *table;
9790         uint32_t n_actions = 0, n_tables = 0;
9791
9792         if (!p || !pipeline)
9793                 return -EINVAL;
9794
9795         TAILQ_FOREACH(action, &p->actions, node)
9796                 n_actions++;
9797
9798         TAILQ_FOREACH(table, &p->tables, node)
9799                 n_tables++;
9800
9801         pipeline->n_ports_in = p->n_ports_in;
9802         pipeline->n_ports_out = p->n_ports_out;
9803         pipeline->n_mirroring_slots = p->n_mirroring_slots;
9804         pipeline->n_mirroring_sessions = p->n_mirroring_sessions;
9805         pipeline->n_actions = n_actions;
9806         pipeline->n_tables = n_tables;
9807         pipeline->n_selectors = p->n_selectors;
9808         pipeline->n_learners = p->n_learners;
9809         pipeline->n_regarrays = p->n_regarrays;
9810         pipeline->n_metarrays = p->n_metarrays;
9811
9812         return 0;
9813 }
9814
9815 int
9816 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9817 {
9818         if (!p || !numa_node)
9819                 return -EINVAL;
9820
9821         *numa_node = p->numa_node;
9822         return 0;
9823 }
9824
9825 int
9826 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9827                             uint32_t action_id,
9828                             struct rte_swx_ctl_action_info *action)
9829 {
9830         struct action *a = NULL;
9831
9832         if (!p || (action_id >= p->n_actions) || !action)
9833                 return -EINVAL;
9834
9835         a = action_find_by_id(p, action_id);
9836         if (!a)
9837                 return -EINVAL;
9838
9839         strcpy(action->name, a->name);
9840         action->n_args = a->st ? a->st->n_fields : 0;
9841         return 0;
9842 }
9843
9844 int
9845 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9846                                 uint32_t action_id,
9847                                 uint32_t action_arg_id,
9848                                 struct rte_swx_ctl_action_arg_info *action_arg)
9849 {
9850         struct action *a = NULL;
9851         struct field *arg = NULL;
9852
9853         if (!p || (action_id >= p->n_actions) || !action_arg)
9854                 return -EINVAL;
9855
9856         a = action_find_by_id(p, action_id);
9857         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9858                 return -EINVAL;
9859
9860         arg = &a->st->fields[action_arg_id];
9861         strcpy(action_arg->name, arg->name);
9862         action_arg->n_bits = arg->n_bits;
9863         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9864
9865         return 0;
9866 }
9867
9868 int
9869 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9870                            uint32_t table_id,
9871                            struct rte_swx_ctl_table_info *table)
9872 {
9873         struct table *t = NULL;
9874
9875         if (!p || !table)
9876                 return -EINVAL;
9877
9878         t = table_find_by_id(p, table_id);
9879         if (!t)
9880                 return -EINVAL;
9881
9882         strcpy(table->name, t->name);
9883         strcpy(table->args, t->args);
9884         table->n_match_fields = t->n_fields;
9885         table->n_actions = t->n_actions;
9886         table->default_action_is_const = t->default_action_is_const;
9887         table->size = t->size;
9888         return 0;
9889 }
9890
9891 int
9892 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9893         uint32_t table_id,
9894         uint32_t match_field_id,
9895         struct rte_swx_ctl_table_match_field_info *match_field)
9896 {
9897         struct table *t;
9898         struct match_field *f;
9899
9900         if (!p || (table_id >= p->n_tables) || !match_field)
9901                 return -EINVAL;
9902
9903         t = table_find_by_id(p, table_id);
9904         if (!t || (match_field_id >= t->n_fields))
9905                 return -EINVAL;
9906
9907         f = &t->fields[match_field_id];
9908         match_field->match_type = f->match_type;
9909         match_field->is_header = t->header ? 1 : 0;
9910         match_field->n_bits = f->field->n_bits;
9911         match_field->offset = f->field->offset;
9912
9913         return 0;
9914 }
9915
9916 int
9917 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9918         uint32_t table_id,
9919         uint32_t table_action_id,
9920         struct rte_swx_ctl_table_action_info *table_action)
9921 {
9922         struct table *t;
9923
9924         if (!p || (table_id >= p->n_tables) || !table_action)
9925                 return -EINVAL;
9926
9927         t = table_find_by_id(p, table_id);
9928         if (!t || (table_action_id >= t->n_actions))
9929                 return -EINVAL;
9930
9931         table_action->action_id = t->actions[table_action_id]->id;
9932
9933         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9934         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9935
9936         return 0;
9937 }
9938
9939 int
9940 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9941                           uint32_t table_id,
9942                           struct rte_swx_table_ops *table_ops,
9943                           int *is_stub)
9944 {
9945         struct table *t;
9946
9947         if (!p || (table_id >= p->n_tables))
9948                 return -EINVAL;
9949
9950         t = table_find_by_id(p, table_id);
9951         if (!t)
9952                 return -EINVAL;
9953
9954         if (t->type) {
9955                 if (table_ops)
9956                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9957                 *is_stub = 0;
9958         } else {
9959                 *is_stub = 1;
9960         }
9961
9962         return 0;
9963 }
9964
9965 int
9966 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9967                               uint32_t selector_id,
9968                               struct rte_swx_ctl_selector_info *selector)
9969 {
9970         struct selector *s = NULL;
9971
9972         if (!p || !selector)
9973                 return -EINVAL;
9974
9975         s = selector_find_by_id(p, selector_id);
9976         if (!s)
9977                 return -EINVAL;
9978
9979         strcpy(selector->name, s->name);
9980
9981         selector->n_selector_fields = s->n_selector_fields;
9982         selector->n_groups_max = s->n_groups_max;
9983         selector->n_members_per_group_max = s->n_members_per_group_max;
9984
9985         return 0;
9986 }
9987
9988 int
9989 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9990          uint32_t selector_id,
9991          struct rte_swx_ctl_table_match_field_info *field)
9992 {
9993         struct selector *s;
9994
9995         if (!p || (selector_id >= p->n_selectors) || !field)
9996                 return -EINVAL;
9997
9998         s = selector_find_by_id(p, selector_id);
9999         if (!s)
10000                 return -EINVAL;
10001
10002         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10003         field->is_header = 0;
10004         field->n_bits = s->group_id_field->n_bits;
10005         field->offset = s->group_id_field->offset;
10006
10007         return 0;
10008 }
10009
10010 int
10011 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10012          uint32_t selector_id,
10013          uint32_t selector_field_id,
10014          struct rte_swx_ctl_table_match_field_info *field)
10015 {
10016         struct selector *s;
10017         struct field *f;
10018
10019         if (!p || (selector_id >= p->n_selectors) || !field)
10020                 return -EINVAL;
10021
10022         s = selector_find_by_id(p, selector_id);
10023         if (!s || (selector_field_id >= s->n_selector_fields))
10024                 return -EINVAL;
10025
10026         f = s->selector_fields[selector_field_id];
10027         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10028         field->is_header = s->selector_header ? 1 : 0;
10029         field->n_bits = f->n_bits;
10030         field->offset = f->offset;
10031
10032         return 0;
10033 }
10034
10035 int
10036 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10037          uint32_t selector_id,
10038          struct rte_swx_ctl_table_match_field_info *field)
10039 {
10040         struct selector *s;
10041
10042         if (!p || (selector_id >= p->n_selectors) || !field)
10043                 return -EINVAL;
10044
10045         s = selector_find_by_id(p, selector_id);
10046         if (!s)
10047                 return -EINVAL;
10048
10049         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10050         field->is_header = 0;
10051         field->n_bits = s->member_id_field->n_bits;
10052         field->offset = s->member_id_field->offset;
10053
10054         return 0;
10055 }
10056
10057 int
10058 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10059                              uint32_t learner_id,
10060                              struct rte_swx_ctl_learner_info *learner)
10061 {
10062         struct learner *l = NULL;
10063
10064         if (!p || !learner)
10065                 return -EINVAL;
10066
10067         l = learner_find_by_id(p, learner_id);
10068         if (!l)
10069                 return -EINVAL;
10070
10071         strcpy(learner->name, l->name);
10072
10073         learner->n_match_fields = l->n_fields;
10074         learner->n_actions = l->n_actions;
10075         learner->default_action_is_const = l->default_action_is_const;
10076         learner->size = l->size;
10077         learner->n_key_timeouts = l->n_timeouts;
10078
10079         return 0;
10080 }
10081
10082 int
10083 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10084                                          uint32_t learner_id,
10085                                          uint32_t match_field_id,
10086                                          struct rte_swx_ctl_table_match_field_info *match_field)
10087 {
10088         struct learner *l;
10089         struct field *f;
10090
10091         if (!p || (learner_id >= p->n_learners) || !match_field)
10092                 return -EINVAL;
10093
10094         l = learner_find_by_id(p, learner_id);
10095         if (!l || (match_field_id >= l->n_fields))
10096                 return -EINVAL;
10097
10098         f = l->fields[match_field_id];
10099         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10100         match_field->is_header = l->header ? 1 : 0;
10101         match_field->n_bits = f->n_bits;
10102         match_field->offset = f->offset;
10103
10104         return 0;
10105 }
10106
10107 int
10108 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10109                                     uint32_t learner_id,
10110                                     uint32_t learner_action_id,
10111                                     struct rte_swx_ctl_table_action_info *learner_action)
10112 {
10113         struct learner *l;
10114
10115         if (!p || (learner_id >= p->n_learners) || !learner_action)
10116                 return -EINVAL;
10117
10118         l = learner_find_by_id(p, learner_id);
10119         if (!l || (learner_action_id >= l->n_actions))
10120                 return -EINVAL;
10121
10122         learner_action->action_id = l->actions[learner_action_id]->id;
10123
10124         learner_action->action_is_for_table_entries =
10125                 l->action_is_for_table_entries[learner_action_id];
10126
10127         learner_action->action_is_for_default_entry =
10128                 l->action_is_for_default_entry[learner_action_id];
10129
10130         return 0;
10131 }
10132
10133 int
10134 rte_swx_ctl_pipeline_learner_timeout_get(struct rte_swx_pipeline *p,
10135                                          uint32_t learner_id,
10136                                          uint32_t timeout_id,
10137                                          uint32_t *timeout)
10138 {
10139         struct learner *l;
10140
10141         if (!p || (learner_id >= p->n_learners) || !timeout)
10142                 return -EINVAL;
10143
10144         l = learner_find_by_id(p, learner_id);
10145         if (!l || (timeout_id >= l->n_timeouts))
10146                 return -EINVAL;
10147
10148         *timeout = l->timeout[timeout_id];
10149         return 0;
10150 }
10151
10152 int
10153 rte_swx_ctl_pipeline_learner_timeout_set(struct rte_swx_pipeline *p,
10154                                          uint32_t learner_id,
10155                                          uint32_t timeout_id,
10156                                          uint32_t timeout)
10157 {
10158         struct learner *l;
10159         struct rte_swx_table_state *ts;
10160         int status;
10161
10162         if (!p || (learner_id >= p->n_learners) || !timeout)
10163                 return -EINVAL;
10164
10165         l = learner_find_by_id(p, learner_id);
10166         if (!l || (timeout_id >= l->n_timeouts))
10167                 return -EINVAL;
10168
10169         if (!p->build_done)
10170                 return -EINVAL;
10171
10172         ts = &p->table_state[p->n_tables + p->n_selectors + l->id];
10173
10174         status = rte_swx_table_learner_timeout_update(ts->obj, timeout_id, timeout);
10175         if (status)
10176                 return -EINVAL;
10177
10178         l->timeout[timeout_id] = timeout;
10179
10180         return 0;
10181 }
10182
10183 int
10184 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10185                                  struct rte_swx_table_state **table_state)
10186 {
10187         if (!p || !table_state || !p->build_done)
10188                 return -EINVAL;
10189
10190         *table_state = p->table_state;
10191         return 0;
10192 }
10193
10194 int
10195 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10196                                  struct rte_swx_table_state *table_state)
10197 {
10198         if (!p || !table_state || !p->build_done)
10199                 return -EINVAL;
10200
10201         p->table_state = table_state;
10202         return 0;
10203 }
10204
10205 int
10206 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10207                                         uint32_t port_id,
10208                                         struct rte_swx_port_in_stats *stats)
10209 {
10210         struct port_in *port;
10211
10212         if (!p || !stats)
10213                 return -EINVAL;
10214
10215         port = port_in_find(p, port_id);
10216         if (!port)
10217                 return -EINVAL;
10218
10219         port->type->ops.stats_read(port->obj, stats);
10220         return 0;
10221 }
10222
10223 int
10224 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10225                                          uint32_t port_id,
10226                                          struct rte_swx_port_out_stats *stats)
10227 {
10228         struct port_out *port;
10229
10230         if (!p || !stats)
10231                 return -EINVAL;
10232
10233         port = port_out_find(p, port_id);
10234         if (!port)
10235                 return -EINVAL;
10236
10237         port->type->ops.stats_read(port->obj, stats);
10238         return 0;
10239 }
10240
10241 int
10242 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10243                                       const char *table_name,
10244                                       struct rte_swx_table_stats *stats)
10245 {
10246         struct table *table;
10247         struct table_statistics *table_stats;
10248
10249         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10250                 return -EINVAL;
10251
10252         table = table_find(p, table_name);
10253         if (!table)
10254                 return -EINVAL;
10255
10256         table_stats = &p->table_stats[table->id];
10257
10258         memcpy(stats->n_pkts_action,
10259                table_stats->n_pkts_action,
10260                p->n_actions * sizeof(uint64_t));
10261
10262         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10263         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10264
10265         return 0;
10266 }
10267
10268 int
10269 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10270         const char *selector_name,
10271         struct rte_swx_pipeline_selector_stats *stats)
10272 {
10273         struct selector *s;
10274
10275         if (!p || !selector_name || !selector_name[0] || !stats)
10276                 return -EINVAL;
10277
10278         s = selector_find(p, selector_name);
10279         if (!s)
10280                 return -EINVAL;
10281
10282         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10283
10284         return 0;
10285 }
10286
10287 int
10288 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10289                                         const char *learner_name,
10290                                         struct rte_swx_learner_stats *stats)
10291 {
10292         struct learner *l;
10293         struct learner_statistics *learner_stats;
10294
10295         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10296                 return -EINVAL;
10297
10298         l = learner_find(p, learner_name);
10299         if (!l)
10300                 return -EINVAL;
10301
10302         learner_stats = &p->learner_stats[l->id];
10303
10304         memcpy(stats->n_pkts_action,
10305                learner_stats->n_pkts_action,
10306                p->n_actions * sizeof(uint64_t));
10307
10308         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10309         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10310
10311         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10312         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10313
10314         stats->n_pkts_rearm = learner_stats->n_pkts_rearm;
10315         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10316
10317         return 0;
10318 }
10319
10320 int
10321 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10322                               uint32_t regarray_id,
10323                               struct rte_swx_ctl_regarray_info *regarray)
10324 {
10325         struct regarray *r;
10326
10327         if (!p || !regarray)
10328                 return -EINVAL;
10329
10330         r = regarray_find_by_id(p, regarray_id);
10331         if (!r)
10332                 return -EINVAL;
10333
10334         strcpy(regarray->name, r->name);
10335         regarray->size = r->size;
10336         return 0;
10337 }
10338
10339 int
10340 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10341                                    const char *regarray_name,
10342                                    uint32_t regarray_index,
10343                                    uint64_t *value)
10344 {
10345         struct regarray *regarray;
10346         struct regarray_runtime *r;
10347
10348         if (!p || !regarray_name || !value)
10349                 return -EINVAL;
10350
10351         regarray = regarray_find(p, regarray_name);
10352         if (!regarray || (regarray_index >= regarray->size))
10353                 return -EINVAL;
10354
10355         r = &p->regarray_runtime[regarray->id];
10356         *value = r->regarray[regarray_index];
10357         return 0;
10358 }
10359
10360 int
10361 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10362                                    const char *regarray_name,
10363                                    uint32_t regarray_index,
10364                                    uint64_t value)
10365 {
10366         struct regarray *regarray;
10367         struct regarray_runtime *r;
10368
10369         if (!p || !regarray_name)
10370                 return -EINVAL;
10371
10372         regarray = regarray_find(p, regarray_name);
10373         if (!regarray || (regarray_index >= regarray->size))
10374                 return -EINVAL;
10375
10376         r = &p->regarray_runtime[regarray->id];
10377         r->regarray[regarray_index] = value;
10378         return 0;
10379 }
10380
10381 int
10382 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10383                               uint32_t metarray_id,
10384                               struct rte_swx_ctl_metarray_info *metarray)
10385 {
10386         struct metarray *m;
10387
10388         if (!p || !metarray)
10389                 return -EINVAL;
10390
10391         m = metarray_find_by_id(p, metarray_id);
10392         if (!m)
10393                 return -EINVAL;
10394
10395         strcpy(metarray->name, m->name);
10396         metarray->size = m->size;
10397         return 0;
10398 }
10399
10400 int
10401 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10402                               const char *name,
10403                               struct rte_meter_trtcm_params *params)
10404 {
10405         struct meter_profile *mp;
10406         int status;
10407
10408         CHECK(p, EINVAL);
10409         CHECK_NAME(name, EINVAL);
10410         CHECK(params, EINVAL);
10411         CHECK(!meter_profile_find(p, name), EEXIST);
10412
10413         /* Node allocation. */
10414         mp = calloc(1, sizeof(struct meter_profile));
10415         CHECK(mp, ENOMEM);
10416
10417         /* Node initialization. */
10418         strcpy(mp->name, name);
10419         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10420         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10421         if (status) {
10422                 free(mp);
10423                 CHECK(0, EINVAL);
10424         }
10425
10426         /* Node add to tailq. */
10427         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10428
10429         return 0;
10430 }
10431
10432 int
10433 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10434                                  const char *name)
10435 {
10436         struct meter_profile *mp;
10437
10438         CHECK(p, EINVAL);
10439         CHECK_NAME(name, EINVAL);
10440
10441         mp = meter_profile_find(p, name);
10442         CHECK(mp, EINVAL);
10443         CHECK(!mp->n_users, EBUSY);
10444
10445         /* Remove node from tailq. */
10446         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10447         free(mp);
10448
10449         return 0;
10450 }
10451
10452 int
10453 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10454                         const char *metarray_name,
10455                         uint32_t metarray_index)
10456 {
10457         struct meter_profile *mp_old;
10458         struct metarray *metarray;
10459         struct metarray_runtime *metarray_runtime;
10460         struct meter *m;
10461
10462         CHECK(p, EINVAL);
10463         CHECK_NAME(metarray_name, EINVAL);
10464
10465         metarray = metarray_find(p, metarray_name);
10466         CHECK(metarray, EINVAL);
10467         CHECK(metarray_index < metarray->size, EINVAL);
10468
10469         metarray_runtime = &p->metarray_runtime[metarray->id];
10470         m = &metarray_runtime->metarray[metarray_index];
10471         mp_old = m->profile;
10472
10473         meter_init(m);
10474
10475         mp_old->n_users--;
10476
10477         return 0;
10478 }
10479
10480 int
10481 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10482                       const char *metarray_name,
10483                       uint32_t metarray_index,
10484                       const char *profile_name)
10485 {
10486         struct meter_profile *mp, *mp_old;
10487         struct metarray *metarray;
10488         struct metarray_runtime *metarray_runtime;
10489         struct meter *m;
10490
10491         CHECK(p, EINVAL);
10492         CHECK_NAME(metarray_name, EINVAL);
10493
10494         metarray = metarray_find(p, metarray_name);
10495         CHECK(metarray, EINVAL);
10496         CHECK(metarray_index < metarray->size, EINVAL);
10497
10498         mp = meter_profile_find(p, profile_name);
10499         CHECK(mp, EINVAL);
10500
10501         metarray_runtime = &p->metarray_runtime[metarray->id];
10502         m = &metarray_runtime->metarray[metarray_index];
10503         mp_old = m->profile;
10504
10505         memset(m, 0, sizeof(struct meter));
10506         rte_meter_trtcm_config(&m->m, &mp->profile);
10507         m->profile = mp;
10508         m->color_mask = RTE_COLORS;
10509
10510         mp->n_users++;
10511         mp_old->n_users--;
10512
10513         return 0;
10514 }
10515
10516 int
10517 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10518                              const char *metarray_name,
10519                              uint32_t metarray_index,
10520                              struct rte_swx_ctl_meter_stats *stats)
10521 {
10522         struct metarray *metarray;
10523         struct metarray_runtime *metarray_runtime;
10524         struct meter *m;
10525
10526         CHECK(p, EINVAL);
10527         CHECK_NAME(metarray_name, EINVAL);
10528
10529         metarray = metarray_find(p, metarray_name);
10530         CHECK(metarray, EINVAL);
10531         CHECK(metarray_index < metarray->size, EINVAL);
10532
10533         CHECK(stats, EINVAL);
10534
10535         metarray_runtime = &p->metarray_runtime[metarray->id];
10536         m = &metarray_runtime->metarray[metarray_index];
10537
10538         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10539         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10540
10541         return 0;
10542 }
10543
10544 int
10545 rte_swx_ctl_pipeline_mirroring_session_set(struct rte_swx_pipeline *p,
10546                                            uint32_t session_id,
10547                                            struct rte_swx_pipeline_mirroring_session_params *params)
10548 {
10549         struct mirroring_session *s;
10550
10551         CHECK(p, EINVAL);
10552         CHECK(p->build_done, EEXIST);
10553         CHECK(session_id < p->n_mirroring_sessions, EINVAL);
10554         CHECK(params, EINVAL);
10555         CHECK(params->port_id < p->n_ports_out, EINVAL);
10556
10557         s = &p->mirroring_sessions[session_id];
10558         s->port_id = params->port_id;
10559         s->fast_clone = params->fast_clone;
10560         s->truncation_length = params->truncation_length ? params->truncation_length : UINT32_MAX;
10561
10562         return 0;
10563 }
10564
10565 /*
10566  * Pipeline compilation.
10567  */
10568 static const char *
10569 instr_type_to_name(struct instruction *instr)
10570 {
10571         switch (instr->type) {
10572         case INSTR_RX: return "INSTR_RX";
10573
10574         case INSTR_TX: return "INSTR_TX";
10575         case INSTR_TX_I: return "INSTR_TX_I";
10576         case INSTR_DROP: return "INSTR_DROP";
10577         case INSTR_MIRROR: return "INSTR_MIRROR";
10578         case INSTR_RECIRCULATE: return "INSTR_RECIRCULATE";
10579         case INSTR_RECIRCID: return "INSTR_RECIRCID";
10580
10581         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10582         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10583         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10584         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10585         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10586         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10587         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10588         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10589
10590         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10591
10592         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10593
10594         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10595         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10596         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10597         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10598         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10599         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10600         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10601         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10602         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10603
10604         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10605         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10606
10607         case INSTR_MOV: return "INSTR_MOV";
10608         case INSTR_MOV_MH: return "INSTR_MOV_MH";
10609         case INSTR_MOV_HM: return "INSTR_MOV_HM";
10610         case INSTR_MOV_HH: return "INSTR_MOV_HH";
10611         case INSTR_MOV_I: return "INSTR_MOV_I";
10612
10613         case INSTR_DMA_HT: return "INSTR_DMA_HT";
10614         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10615         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10616         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10617         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10618         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10619         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10620         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10621
10622         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10623         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10624         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10625         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10626         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10627         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10628
10629         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10630         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10631         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10632         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10633         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10634         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10635
10636         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10637         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10638         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10639         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10640
10641         case INSTR_ALU_AND: return "INSTR_ALU_AND";
10642         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10643         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10644         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10645         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10646
10647         case INSTR_ALU_OR: return "INSTR_ALU_OR";
10648         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10649         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10650         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10651         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10652
10653         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10654         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10655         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10656         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10657         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10658
10659         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10660         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10661         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10662         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10663         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10664         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10665
10666         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10667         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10668         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10669         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10670         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10671         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10672
10673         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10674         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10675         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10676
10677         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10678         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10679         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10680         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10681         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10682         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10683
10684         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10685         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10686         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10687         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10688         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10689         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10690         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10691         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10692         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10693
10694         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10695         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10696         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10697         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10698         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10699         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10700         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10701         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10702         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10703
10704         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10705         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10706         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10707
10708         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10709         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10710         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10711         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10712         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10713         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10714         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10715         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10716         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10717         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10718         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10719         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10720
10721         case INSTR_TABLE: return "INSTR_TABLE";
10722         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10723         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10724         case INSTR_LEARNER: return "INSTR_LEARNER";
10725         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10726
10727         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10728         case INSTR_LEARNER_REARM: return "INSTR_LEARNER_REARM";
10729         case INSTR_LEARNER_REARM_NEW: return "INSTR_LEARNER_REARM_NEW";
10730         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10731
10732         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10733         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10734
10735         case INSTR_JMP: return "INSTR_JMP";
10736         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10737         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10738         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10739         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10740         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10741         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10742         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10743         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10744         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10745         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10746         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10747         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10748         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10749         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10750         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10751         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10752         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10753         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10754         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10755         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10756         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10757         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10758         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10759         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10760         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10761         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10762         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10763         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10764
10765         case INSTR_RETURN: return "INSTR_RETURN";
10766
10767         default: return "INSTR_UNKNOWN";
10768         }
10769 }
10770
10771 typedef void
10772 (*instruction_export_t)(struct instruction *, FILE *);
10773
10774 static void
10775 instr_io_export(struct instruction *instr, FILE *f)
10776 {
10777         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10778
10779         /* n_io, n_io_imm, n_hdrs. */
10780         if (instr->type == INSTR_RX ||
10781             instr->type == INSTR_TX ||
10782             instr->type == INSTR_HDR_EXTRACT_M ||
10783             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10784                 n_io = 1;
10785
10786         if (instr->type == INSTR_TX_I)
10787                 n_io_imm = 1;
10788
10789         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10790                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10791
10792         if (instr->type == INSTR_HDR_EXTRACT_M ||
10793             instr->type == INSTR_HDR_LOOKAHEAD ||
10794             instr->type == INSTR_HDR_EMIT)
10795                 n_hdrs = 1;
10796
10797         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10798                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10799
10800         /* instr. */
10801         fprintf(f,
10802                 "\t{\n"
10803                 "\t\t.type = %s,\n",
10804                 instr_type_to_name(instr));
10805
10806         /* instr.io. */
10807         if (n_io || n_io_imm || n_hdrs)
10808                 fprintf(f,
10809                         "\t\t.io = {\n");
10810
10811         /* instr.io.io. */
10812         if (n_io)
10813                 fprintf(f,
10814                         "\t\t\t.io = {\n"
10815                         "\t\t\t\t.offset = %u,\n"
10816                         "\t\t\t\t.n_bits = %u,\n"
10817                         "\t\t\t},\n",
10818                         instr->io.io.offset,
10819                         instr->io.io.n_bits);
10820
10821         if (n_io_imm)
10822                 fprintf(f,
10823                         "\t\t\t.io = {\n"
10824                         "\t\t\t\t.val = %u,\n"
10825                         "\t\t\t},\n",
10826                         instr->io.io.val);
10827
10828         /* instr.io.hdr. */
10829         if (n_hdrs) {
10830                 fprintf(f,
10831                         "\t\t.hdr = {\n");
10832
10833                 /* instr.io.hdr.header_id. */
10834                 fprintf(f,
10835                         "\t\t\t.header_id = {");
10836
10837                 for (i = 0; i < n_hdrs; i++)
10838                         fprintf(f,
10839                                 "%u, ",
10840                                 instr->io.hdr.header_id[i]);
10841
10842                 fprintf(f,
10843                         "},\n");
10844
10845                 /* instr.io.hdr.struct_id. */
10846                 fprintf(f,
10847                         "\t\t\t.struct_id = {");
10848
10849                 for (i = 0; i < n_hdrs; i++)
10850                         fprintf(f,
10851                                 "%u, ",
10852                                 instr->io.hdr.struct_id[i]);
10853
10854                 fprintf(f,
10855                         "},\n");
10856
10857                 /* instr.io.hdr.n_bytes. */
10858                 fprintf(f,
10859                         "\t\t\t.n_bytes = {");
10860
10861                 for (i = 0; i < n_hdrs; i++)
10862                         fprintf(f,
10863                                 "%u, ",
10864                                 instr->io.hdr.n_bytes[i]);
10865
10866                 fprintf(f,
10867                         "},\n");
10868
10869                 /* instr.io.hdr - closing curly brace. */
10870                 fprintf(f,
10871                         "\t\t\t}\n,");
10872         }
10873
10874         /* instr.io - closing curly brace. */
10875         if (n_io || n_io_imm || n_hdrs)
10876                 fprintf(f,
10877                         "\t\t},\n");
10878
10879         /* instr - closing curly brace. */
10880         fprintf(f,
10881                 "\t},\n");
10882 }
10883
10884 static void
10885 instr_mirror_export(struct instruction *instr, FILE *f)
10886 {
10887         fprintf(f,
10888                 "\t{\n"
10889                 "\t\t.type = %s,\n"
10890                 "\t\t.mirror = {\n"
10891                 "\t\t\t.dst = {\n"
10892                 "\t\t\t\t.struct_id = %u,\n"
10893                 "\t\t\t\t.n_bits = %u,\n"
10894                 "\t\t\t\t.offset = %u,\n"
10895                 "\t\t\t}\n,"
10896                 "\t\t\t.src = {\n"
10897                 "\t\t\t\t.struct_id = %u,\n"
10898                 "\t\t\t\t.n_bits = %u,\n"
10899                 "\t\t\t\t.offset = %u,\n"
10900                 "\t\t\t}\n,"
10901                 "\t\t},\n"
10902                 "\t},\n",
10903                 instr_type_to_name(instr),
10904                 instr->mirror.dst.struct_id,
10905                 instr->mirror.dst.n_bits,
10906                 instr->mirror.dst.offset,
10907                 instr->mirror.src.struct_id,
10908                 instr->mirror.src.n_bits,
10909                 instr->mirror.src.offset);
10910 }
10911
10912 static void
10913 instr_recirculate_export(struct instruction *instr, FILE *f)
10914 {
10915         fprintf(f,
10916                 "\t{\n"
10917                 "\t\t.type = %s,\n"
10918                 "\t},\n",
10919                 instr_type_to_name(instr));
10920 }
10921
10922 static void
10923 instr_recircid_export(struct instruction *instr, FILE *f)
10924 {
10925         fprintf(f,
10926                 "\t{\n"
10927                 "\t\t.type = %s,\n"
10928                 "\t\t.io = {\n"
10929                 "\t\t\t.offset = %u,\n"
10930                 "\t\t\t.n_bits = %u,\n"
10931                 "\t\t},\n"
10932                 "\t},\n",
10933                 instr_type_to_name(instr),
10934                 instr->io.io.offset,
10935                 instr->io.io.n_bits);
10936 }
10937
10938 static void
10939 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10940 {
10941         fprintf(f,
10942                 "\t{\n"
10943                 "\t\t.type = %s,\n"
10944                 "\t\t.valid = {\n"
10945                 "\t\t\t.header_id = %u,\n"
10946                 "\t\t},\n"
10947                 "\t},\n",
10948                 instr_type_to_name(instr),
10949                 instr->valid.header_id);
10950 }
10951
10952 static void
10953 instr_mov_export(struct instruction *instr, FILE *f)
10954 {
10955         if (instr->type != INSTR_MOV_I)
10956                 fprintf(f,
10957                         "\t{\n"
10958                         "\t\t.type = %s,\n"
10959                         "\t\t.mov = {\n"
10960                         "\t\t\t.dst = {\n"
10961                         "\t\t\t\t.struct_id = %u,\n"
10962                         "\t\t\t\t.n_bits = %u,\n"
10963                         "\t\t\t\t.offset = %u,\n"
10964                         "\t\t\t},\n"
10965                         "\t\t\t.src = {\n"
10966                         "\t\t\t\t.struct_id = %u,\n"
10967                         "\t\t\t\t.n_bits = %u,\n"
10968                         "\t\t\t\t.offset = %u,\n"
10969                         "\t\t\t},\n"
10970                         "\t\t},\n"
10971                         "\t},\n",
10972                         instr_type_to_name(instr),
10973                         instr->mov.dst.struct_id,
10974                         instr->mov.dst.n_bits,
10975                         instr->mov.dst.offset,
10976                         instr->mov.src.struct_id,
10977                         instr->mov.src.n_bits,
10978                         instr->mov.src.offset);
10979         else
10980                 fprintf(f,
10981                         "\t{\n"
10982                         "\t\t.type = %s,\n"
10983                         "\t\t.mov = {\n"
10984                         "\t\t\t.dst = {\n"
10985                         "\t\t\t\t.struct_id = %u,\n"
10986                         "\t\t\t\t.n_bits = %u,\n"
10987                         "\t\t\t\t.offset = %u,\n"
10988                         "\t\t\t}\n,"
10989                         "\t\t\t.src_val = %" PRIu64 ",\n"
10990                         "\t\t},\n"
10991                         "\t},\n",
10992                         instr_type_to_name(instr),
10993                         instr->mov.dst.struct_id,
10994                         instr->mov.dst.n_bits,
10995                         instr->mov.dst.offset,
10996                         instr->mov.src_val);
10997 }
10998
10999 static void
11000 instr_dma_ht_export(struct instruction *instr, FILE *f)
11001 {
11002         uint32_t n_dma = 0, i;
11003
11004         /* n_dma. */
11005         n_dma = 1 + (instr->type - INSTR_DMA_HT);
11006
11007         /* instr. */
11008         fprintf(f,
11009                 "\t{\n"
11010                 "\t\t.type = %s,\n",
11011                 instr_type_to_name(instr));
11012
11013         /* instr.dma. */
11014         fprintf(f,
11015                 "\t\t.dma = {\n");
11016
11017         /* instr.dma.dst. */
11018         fprintf(f,
11019                 "\t\t\t.dst = {\n");
11020
11021         /* instr.dma.dst.header_id. */
11022         fprintf(f,
11023                 "\t\t\t\t.header_id = {");
11024
11025         for (i = 0; i < n_dma; i++)
11026                 fprintf(f,
11027                         "%u, ",
11028                         instr->dma.dst.header_id[i]);
11029
11030         fprintf(f,
11031                 "},\n");
11032
11033         /* instr.dma.dst.struct_id. */
11034         fprintf(f,
11035                 "\t\t\t\t.struct_id = {");
11036
11037         for (i = 0; i < n_dma; i++)
11038                 fprintf(f,
11039                         "%u, ",
11040                         instr->dma.dst.struct_id[i]);
11041
11042         fprintf(f,
11043                 "},\n");
11044
11045         /* instr.dma.dst - closing curly brace. */
11046         fprintf(f,
11047                 "\t\t\t},\n");
11048
11049         /* instr.dma.src. */
11050         fprintf(f,
11051                 "\t\t\t.src = {\n");
11052
11053         /* instr.dma.src.offset. */
11054         fprintf(f,
11055                 "\t\t\t\t.offset = {");
11056
11057         for (i = 0; i < n_dma; i++)
11058                 fprintf(f,
11059                         "%u, ",
11060                         instr->dma.src.offset[i]);
11061
11062         fprintf(f,
11063                 "},\n");
11064
11065         /* instr.dma.src - closing curly brace. */
11066         fprintf(f,
11067                 "\t\t\t},\n");
11068
11069         /* instr.dma.n_bytes. */
11070         fprintf(f,
11071                 "\t\t\t.n_bytes = {");
11072
11073         for (i = 0; i < n_dma; i++)
11074                 fprintf(f,
11075                         "%u, ",
11076                         instr->dma.n_bytes[i]);
11077
11078         fprintf(f,
11079                 "},\n");
11080
11081         /* instr.dma - closing curly brace. */
11082         fprintf(f,
11083                 "\t\t},\n");
11084
11085         /* instr - closing curly brace. */
11086         fprintf(f,
11087                 "\t},\n");
11088 }
11089
11090 static void
11091 instr_alu_export(struct instruction *instr, FILE *f)
11092 {
11093         int imm = 0;
11094
11095         if (instr->type == INSTR_ALU_ADD_MI ||
11096             instr->type == INSTR_ALU_ADD_HI ||
11097             instr->type == INSTR_ALU_SUB_MI ||
11098             instr->type == INSTR_ALU_SUB_HI ||
11099             instr->type == INSTR_ALU_SHL_MI ||
11100             instr->type == INSTR_ALU_SHL_HI ||
11101             instr->type == INSTR_ALU_SHR_MI ||
11102             instr->type == INSTR_ALU_SHR_HI ||
11103             instr->type == INSTR_ALU_AND_I ||
11104             instr->type == INSTR_ALU_OR_I ||
11105             instr->type == INSTR_ALU_XOR_I)
11106                 imm = 1;
11107
11108         if (!imm)
11109                 fprintf(f,
11110                         "\t{\n"
11111                         "\t\t.type = %s,\n"
11112                         "\t\t.alu = {\n"
11113                         "\t\t\t.dst = {\n"
11114                         "\t\t\t\t.struct_id = %u,\n"
11115                         "\t\t\t\t.n_bits = %u,\n"
11116                         "\t\t\t\t.offset = %u,\n"
11117                         "\t\t\t},\n"
11118                         "\t\t\t.src = {\n"
11119                         "\t\t\t\t.struct_id = %u,\n"
11120                         "\t\t\t\t.n_bits = %u,\n"
11121                         "\t\t\t\t.offset = %u,\n"
11122                         "\t\t\t},\n"
11123                         "\t\t},\n"
11124                         "\t},\n",
11125                         instr_type_to_name(instr),
11126                         instr->alu.dst.struct_id,
11127                         instr->alu.dst.n_bits,
11128                         instr->alu.dst.offset,
11129                         instr->alu.src.struct_id,
11130                         instr->alu.src.n_bits,
11131                         instr->alu.src.offset);
11132         else
11133                 fprintf(f,
11134                         "\t{\n"
11135                         "\t\t.type = %s,\n"
11136                         "\t\t.alu = {\n"
11137                         "\t\t\t.dst = {\n"
11138                         "\t\t\t\t.struct_id = %u,\n"
11139                         "\t\t\t\t.n_bits = %u,\n"
11140                         "\t\t\t\t.offset = %u,\n"
11141                         "\t\t\t}\n,"
11142                         "\t\t\t.src_val = %" PRIu64 ",\n"
11143                         "\t\t},\n"
11144                         "\t},\n",
11145                         instr_type_to_name(instr),
11146                         instr->alu.dst.struct_id,
11147                         instr->alu.dst.n_bits,
11148                         instr->alu.dst.offset,
11149                         instr->alu.src_val);
11150 }
11151
11152 static void
11153 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11154 {
11155         int prefetch  = 0, idx_imm = 0, src_imm = 0;
11156
11157         if (instr->type == INSTR_REGPREFETCH_RH ||
11158             instr->type == INSTR_REGPREFETCH_RM ||
11159             instr->type == INSTR_REGPREFETCH_RI)
11160                 prefetch = 1;
11161
11162         /* index is the 3rd operand for the regrd instruction and the 2nd
11163          * operand for the regwr and regadd instructions.
11164          */
11165         if (instr->type == INSTR_REGPREFETCH_RI ||
11166             instr->type == INSTR_REGRD_HRI ||
11167             instr->type == INSTR_REGRD_MRI ||
11168             instr->type == INSTR_REGWR_RIH ||
11169             instr->type == INSTR_REGWR_RIM ||
11170             instr->type == INSTR_REGWR_RII ||
11171             instr->type == INSTR_REGADD_RIH ||
11172             instr->type == INSTR_REGADD_RIM ||
11173             instr->type == INSTR_REGADD_RII)
11174                 idx_imm = 1;
11175
11176         /* src is the 3rd operand for the regwr and regadd instructions. */
11177         if (instr->type == INSTR_REGWR_RHI ||
11178             instr->type == INSTR_REGWR_RMI ||
11179             instr->type == INSTR_REGWR_RII ||
11180             instr->type == INSTR_REGADD_RHI ||
11181             instr->type == INSTR_REGADD_RMI ||
11182             instr->type == INSTR_REGADD_RII)
11183                 src_imm = 1;
11184
11185         /* instr.regarray.regarray_id. */
11186         fprintf(f,
11187                 "\t{\n"
11188                 "\t\t.type = %s,\n"
11189                 "\t\t.regarray = {\n"
11190                 "\t\t\t.regarray_id = %u,\n",
11191                 instr_type_to_name(instr),
11192                 instr->regarray.regarray_id);
11193
11194         /* instr.regarray.idx / instr.regarray.idx_val. */
11195         if (!idx_imm)
11196                 fprintf(f,
11197                         "\t\t\t\t.idx = {\n"
11198                         "\t\t\t\t\t.struct_id = %u,\n"
11199                         "\t\t\t\t\t.n_bits = %u,\n"
11200                         "\t\t\t\t\t.offset = %u,\n"
11201                         "\t\t\t\t},\n",
11202                         instr->regarray.idx.struct_id,
11203                         instr->regarray.idx.n_bits,
11204                         instr->regarray.idx.offset);
11205         else
11206                 fprintf(f,
11207                         "\t\t\t\t.idx_val = %u,\n",
11208                         instr->regarray.idx_val);
11209
11210         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
11211         if (!prefetch) {
11212                 if (!src_imm)
11213                         fprintf(f,
11214                                 "\t\t\t\t.dstsrc = {\n"
11215                                 "\t\t\t\t\t.struct_id = %u,\n"
11216                                 "\t\t\t\t\t.n_bits = %u,\n"
11217                                 "\t\t\t\t\t.offset = %u,\n"
11218                                 "\t\t\t\t},\n",
11219                                 instr->regarray.dstsrc.struct_id,
11220                                 instr->regarray.dstsrc.n_bits,
11221                                 instr->regarray.dstsrc.offset);
11222                 else
11223                         fprintf(f,
11224                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
11225                                 instr->regarray.dstsrc_val);
11226         }
11227
11228         /* instr.regarray and instr - closing curly braces. */
11229         fprintf(f,
11230                 "\t\t},\n"
11231                 "\t},\n");
11232 }
11233
11234 static void
11235 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11236 {
11237         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
11238
11239         if (instr->type == INSTR_METPREFETCH_H ||
11240             instr->type == INSTR_METPREFETCH_M ||
11241             instr->type == INSTR_METPREFETCH_I)
11242                 prefetch = 1;
11243
11244         /* idx_imm. */
11245         if (instr->type == INSTR_METPREFETCH_I ||
11246             instr->type == INSTR_METER_IHM ||
11247             instr->type == INSTR_METER_IHI ||
11248             instr->type == INSTR_METER_IMM ||
11249             instr->type == INSTR_METER_IMI)
11250                 idx_imm = 1;
11251
11252         /* color_in_imm. */
11253         if (instr->type == INSTR_METER_HHI ||
11254             instr->type == INSTR_METER_HMI ||
11255             instr->type == INSTR_METER_MHI ||
11256             instr->type == INSTR_METER_MMI ||
11257             instr->type == INSTR_METER_IHI ||
11258             instr->type == INSTR_METER_IMI)
11259                 color_in_imm = 1;
11260
11261         /* instr.meter.metarray_id. */
11262         fprintf(f,
11263                 "\t{\n"
11264                 "\t\t.type = %s,\n"
11265                 "\t\t.meter = {\n"
11266                 "\t\t\t.metarray_id = %u,\n",
11267                 instr_type_to_name(instr),
11268                 instr->meter.metarray_id);
11269
11270         /* instr.meter.idx / instr.meter.idx_val. */
11271         if (!idx_imm)
11272                 fprintf(f,
11273                         "\t\t\t.idx = {\n"
11274                         "\t\t\t\t.struct_id = %u,\n"
11275                         "\t\t\t\t.n_bits = %u,\n"
11276                         "\t\t\t\t.offset = %u,\n"
11277                         "\t\t\t},\n",
11278                         instr->meter.idx.struct_id,
11279                         instr->meter.idx.n_bits,
11280                         instr->meter.idx.offset);
11281         else
11282                 fprintf(f,
11283                         "\t\t\t.idx_val = %u,\n",
11284                         instr->meter.idx_val);
11285
11286         if (!prefetch) {
11287                 /* instr.meter.length. */
11288                 fprintf(f,
11289                         "\t\t\t.length = {\n"
11290                         "\t\t\t\t.struct_id = %u,\n"
11291                         "\t\t\t\t.n_bits = %u,\n"
11292                         "\t\t\t\t.offset = %u,\n"
11293                         "\t\t\t},\n",
11294                         instr->meter.length.struct_id,
11295                         instr->meter.length.n_bits,
11296                         instr->meter.length.offset);
11297
11298                 /* instr.meter.color_in / instr.meter.color_in_val. */
11299                 if (!color_in_imm)
11300                         fprintf(f,
11301                                 "\t\t\t.color_in = {\n"
11302                                 "\t\t\t\t.struct_id = %u,\n"
11303                                 "\t\t\t\t.n_bits = %u,\n"
11304                                 "\t\t\t\t.offset = %u,\n"
11305                                 "\t\t\t},\n",
11306                                 instr->meter.color_in.struct_id,
11307                                 instr->meter.color_in.n_bits,
11308                                 instr->meter.color_in.offset);
11309                 else
11310                         fprintf(f,
11311                                 "\t\t\t.color_in_val = %u,\n",
11312                                 (uint32_t)instr->meter.color_in_val);
11313
11314                 /* instr.meter.color_out. */
11315                 fprintf(f,
11316                         "\t\t\t.color_out = {\n"
11317                         "\t\t\t\t.struct_id = %u,\n"
11318                         "\t\t\t\t.n_bits = %u,\n"
11319                         "\t\t\t\t.offset = %u,\n"
11320                         "\t\t\t},\n",
11321                         instr->meter.color_out.struct_id,
11322                         instr->meter.color_out.n_bits,
11323                         instr->meter.color_out.offset);
11324         }
11325
11326         /* instr.meter and instr - closing curly braces. */
11327         fprintf(f,
11328                 "\t\t},\n"
11329                 "\t},\n");
11330 }
11331
11332 static void
11333 instr_table_export(struct instruction *instr,
11334                 FILE *f)
11335 {
11336         fprintf(f,
11337                 "\t{\n"
11338                 "\t\t.type = %s,\n"
11339                 "\t\t.table = {\n"
11340                 "\t\t\t.table_id = %u,\n"
11341                 "\t\t},\n"
11342                 "\t},\n",
11343                 instr_type_to_name(instr),
11344                 instr->table.table_id);
11345 }
11346
11347 static void
11348 instr_learn_export(struct instruction *instr, FILE *f)
11349 {
11350         fprintf(f,
11351                 "\t{\n"
11352                 "\t\t.type = %s,\n"
11353                 "\t\t.learn = {\n"
11354                 "\t\t\t.action_id = %u,\n"
11355                 "\t\t\t.mf_first_arg_offset = %u,\n"
11356                 "\t\t\t.mf_timeout_id_offset = %u,\n"
11357                 "\t\t\t.mf_timeout_id_n_bits = %u,\n"
11358                 "\t\t},\n"
11359                 "\t},\n",
11360                 instr_type_to_name(instr),
11361                 instr->learn.action_id,
11362                 instr->learn.mf_first_arg_offset,
11363                 instr->learn.mf_timeout_id_offset,
11364                 instr->learn.mf_timeout_id_n_bits);
11365 }
11366
11367 static void
11368 instr_rearm_export(struct instruction *instr, FILE *f)
11369 {
11370         if (instr->type == INSTR_LEARNER_REARM)
11371                 fprintf(f,
11372                         "\t{\n"
11373                         "\t\t.type = %s,\n"
11374                         "\t},\n",
11375                         instr_type_to_name(instr));
11376         else
11377                 fprintf(f,
11378                         "\t{\n"
11379                         "\t\t.type = %s,\n"
11380                         "\t\t.learn = {\n"
11381                         "\t\t\t.mf_timeout_id_offset = %u,\n"
11382                         "\t\t\t.mf_timeout_id_n_bits = %u,\n"
11383                         "\t\t},\n"
11384                         "\t},\n",
11385                         instr_type_to_name(instr),
11386                         instr->learn.mf_timeout_id_offset,
11387                         instr->learn.mf_timeout_id_n_bits);
11388 }
11389
11390 static void
11391 instr_forget_export(struct instruction *instr, FILE *f)
11392 {
11393         fprintf(f,
11394                 "\t{\n"
11395                 "\t\t.type = %s,\n"
11396                 "\t},\n",
11397                 instr_type_to_name(instr));
11398 }
11399
11400 static void
11401 instr_extern_export(struct instruction *instr, FILE *f)
11402 {
11403         if (instr->type == INSTR_EXTERN_OBJ)
11404                 fprintf(f,
11405                         "\t{\n"
11406                         "\t\t.type = %s,\n"
11407                         "\t\t.ext_obj = {\n"
11408                         "\t\t\t.ext_obj_id = %u,\n"
11409                         "\t\t\t.func_id = %u,\n"
11410                         "\t\t},\n"
11411                         "\t},\n",
11412                         instr_type_to_name(instr),
11413                         instr->ext_obj.ext_obj_id,
11414                         instr->ext_obj.func_id);
11415         else
11416                 fprintf(f,
11417                         "\t{\n"
11418                         "\t\t.type = %s,\n"
11419                         "\t\t.ext_func = {\n"
11420                         "\t\t\t.ext_func_id = %u,\n"
11421                         "\t\t},\n"
11422                         "\t},\n",
11423                         instr_type_to_name(instr),
11424                         instr->ext_func.ext_func_id);
11425 }
11426
11427 static void
11428 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
11429 {
11430         fprintf(f,
11431                 "\t{\n"
11432                 "\t\t.type = %s,\n"
11433                 "\t\t.jmp = {\n"
11434                 "\t\t\t.ip = NULL,\n",
11435                 instr_type_to_name(instr));
11436
11437         switch (instr->type) {
11438         case INSTR_JMP_VALID:
11439         case INSTR_JMP_INVALID:
11440                 fprintf(f,
11441                         "\t\t\t.header_id = %u,\n",
11442                         instr->jmp.header_id);
11443                 break;
11444
11445         case INSTR_JMP_ACTION_HIT:
11446         case INSTR_JMP_ACTION_MISS:
11447                 fprintf(f,
11448                         "\t\t\t.action_id = %u,\n",
11449                         instr->jmp.action_id);
11450                 break;
11451
11452         case INSTR_JMP_EQ:
11453         case INSTR_JMP_EQ_MH:
11454         case INSTR_JMP_EQ_HM:
11455         case INSTR_JMP_EQ_HH:
11456         case INSTR_JMP_NEQ:
11457         case INSTR_JMP_NEQ_MH:
11458         case INSTR_JMP_NEQ_HM:
11459         case INSTR_JMP_NEQ_HH:
11460         case INSTR_JMP_LT:
11461         case INSTR_JMP_LT_MH:
11462         case INSTR_JMP_LT_HM:
11463         case INSTR_JMP_LT_HH:
11464         case INSTR_JMP_GT:
11465         case INSTR_JMP_GT_MH:
11466         case INSTR_JMP_GT_HM:
11467         case INSTR_JMP_GT_HH:
11468                 fprintf(f,
11469                         "\t\t\t.a = {\n"
11470                         "\t\t\t\t.struct_id = %u,\n"
11471                         "\t\t\t\t.n_bits = %u,\n"
11472                         "\t\t\t\t.offset = %u,\n"
11473                         "\t\t\t},\n"
11474                         "\t\t\t.b = {\n"
11475                         "\t\t\t\t.struct_id = %u,\n"
11476                         "\t\t\t\t.n_bits = %u,\n"
11477                         "\t\t\t\t.offset = %u,\n"
11478                         "\t\t\t},\n",
11479                         instr->jmp.a.struct_id,
11480                         instr->jmp.a.n_bits,
11481                         instr->jmp.a.offset,
11482                         instr->jmp.b.struct_id,
11483                         instr->jmp.b.n_bits,
11484                         instr->jmp.b.offset);
11485                 break;
11486
11487         case INSTR_JMP_EQ_I:
11488         case INSTR_JMP_NEQ_I:
11489         case INSTR_JMP_LT_MI:
11490         case INSTR_JMP_LT_HI:
11491         case INSTR_JMP_GT_MI:
11492         case INSTR_JMP_GT_HI:
11493                 fprintf(f,
11494                         "\t\t\t.a = {\n"
11495                         "\t\t\t\t.struct_id = %u,\n"
11496                         "\t\t\t\t.n_bits = %u,\n"
11497                         "\t\t\t\t.offset = %u,\n"
11498                         "\t\t\t}\n,"
11499                         "\t\t\t.b_val = %" PRIu64 ",\n",
11500                         instr->jmp.a.struct_id,
11501                         instr->jmp.a.n_bits,
11502                         instr->jmp.a.offset,
11503                         instr->jmp.b_val);
11504                 break;
11505
11506         default:
11507                 break;
11508         }
11509
11510         fprintf(f,
11511                 "\t\t},\n"
11512                 "\t},\n");
11513 }
11514
11515 static void
11516 instr_return_export(struct instruction *instr,
11517                 FILE *f)
11518 {
11519         fprintf(f,
11520                 "\t{\n"
11521                 "\t\t.type = %s,\n",
11522                 instr_type_to_name(instr));
11523
11524         fprintf(f,
11525                 "\t},\n");
11526 }
11527
11528 static instruction_export_t export_table[] = {
11529         [INSTR_RX] = instr_io_export,
11530
11531         [INSTR_TX] = instr_io_export,
11532         [INSTR_TX_I] = instr_io_export,
11533         [INSTR_DROP] = instr_io_export,
11534         [INSTR_MIRROR] = instr_mirror_export,
11535         [INSTR_RECIRCULATE] = instr_recirculate_export,
11536         [INSTR_RECIRCID] = instr_recircid_export,
11537
11538         [INSTR_HDR_EXTRACT] = instr_io_export,
11539         [INSTR_HDR_EXTRACT2] = instr_io_export,
11540         [INSTR_HDR_EXTRACT3] = instr_io_export,
11541         [INSTR_HDR_EXTRACT4] = instr_io_export,
11542         [INSTR_HDR_EXTRACT5] = instr_io_export,
11543         [INSTR_HDR_EXTRACT6] = instr_io_export,
11544         [INSTR_HDR_EXTRACT7] = instr_io_export,
11545         [INSTR_HDR_EXTRACT8] = instr_io_export,
11546
11547         [INSTR_HDR_EXTRACT_M] = instr_io_export,
11548
11549         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
11550
11551         [INSTR_HDR_EMIT] = instr_io_export,
11552         [INSTR_HDR_EMIT_TX] = instr_io_export,
11553         [INSTR_HDR_EMIT2_TX] = instr_io_export,
11554         [INSTR_HDR_EMIT3_TX] = instr_io_export,
11555         [INSTR_HDR_EMIT4_TX] = instr_io_export,
11556         [INSTR_HDR_EMIT5_TX] = instr_io_export,
11557         [INSTR_HDR_EMIT6_TX] = instr_io_export,
11558         [INSTR_HDR_EMIT7_TX] = instr_io_export,
11559         [INSTR_HDR_EMIT8_TX] = instr_io_export,
11560
11561         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
11562         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
11563
11564         [INSTR_MOV] = instr_mov_export,
11565         [INSTR_MOV_MH] = instr_mov_export,
11566         [INSTR_MOV_HM] = instr_mov_export,
11567         [INSTR_MOV_HH] = instr_mov_export,
11568         [INSTR_MOV_I] = instr_mov_export,
11569
11570         [INSTR_DMA_HT]  = instr_dma_ht_export,
11571         [INSTR_DMA_HT2] = instr_dma_ht_export,
11572         [INSTR_DMA_HT3] = instr_dma_ht_export,
11573         [INSTR_DMA_HT4] = instr_dma_ht_export,
11574         [INSTR_DMA_HT5] = instr_dma_ht_export,
11575         [INSTR_DMA_HT6] = instr_dma_ht_export,
11576         [INSTR_DMA_HT7] = instr_dma_ht_export,
11577         [INSTR_DMA_HT8] = instr_dma_ht_export,
11578
11579         [INSTR_ALU_ADD] = instr_alu_export,
11580         [INSTR_ALU_ADD_MH] = instr_alu_export,
11581         [INSTR_ALU_ADD_HM] = instr_alu_export,
11582         [INSTR_ALU_ADD_HH] = instr_alu_export,
11583         [INSTR_ALU_ADD_MI] = instr_alu_export,
11584         [INSTR_ALU_ADD_HI] = instr_alu_export,
11585
11586         [INSTR_ALU_SUB] = instr_alu_export,
11587         [INSTR_ALU_SUB_MH] = instr_alu_export,
11588         [INSTR_ALU_SUB_HM] = instr_alu_export,
11589         [INSTR_ALU_SUB_HH] = instr_alu_export,
11590         [INSTR_ALU_SUB_MI] = instr_alu_export,
11591         [INSTR_ALU_SUB_HI] = instr_alu_export,
11592
11593         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
11594         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
11595         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
11596         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
11597
11598         [INSTR_ALU_AND] = instr_alu_export,
11599         [INSTR_ALU_AND_MH] = instr_alu_export,
11600         [INSTR_ALU_AND_HM] = instr_alu_export,
11601         [INSTR_ALU_AND_HH] = instr_alu_export,
11602         [INSTR_ALU_AND_I] = instr_alu_export,
11603
11604         [INSTR_ALU_OR] = instr_alu_export,
11605         [INSTR_ALU_OR_MH] = instr_alu_export,
11606         [INSTR_ALU_OR_HM] = instr_alu_export,
11607         [INSTR_ALU_OR_HH] = instr_alu_export,
11608         [INSTR_ALU_OR_I] = instr_alu_export,
11609
11610         [INSTR_ALU_XOR] = instr_alu_export,
11611         [INSTR_ALU_XOR_MH] = instr_alu_export,
11612         [INSTR_ALU_XOR_HM] = instr_alu_export,
11613         [INSTR_ALU_XOR_HH] = instr_alu_export,
11614         [INSTR_ALU_XOR_I] = instr_alu_export,
11615
11616         [INSTR_ALU_SHL] = instr_alu_export,
11617         [INSTR_ALU_SHL_MH] = instr_alu_export,
11618         [INSTR_ALU_SHL_HM] = instr_alu_export,
11619         [INSTR_ALU_SHL_HH] = instr_alu_export,
11620         [INSTR_ALU_SHL_MI] = instr_alu_export,
11621         [INSTR_ALU_SHL_HI] = instr_alu_export,
11622
11623         [INSTR_ALU_SHR] = instr_alu_export,
11624         [INSTR_ALU_SHR_MH] = instr_alu_export,
11625         [INSTR_ALU_SHR_HM] = instr_alu_export,
11626         [INSTR_ALU_SHR_HH] = instr_alu_export,
11627         [INSTR_ALU_SHR_MI] = instr_alu_export,
11628         [INSTR_ALU_SHR_HI] = instr_alu_export,
11629
11630         [INSTR_REGPREFETCH_RH] = instr_reg_export,
11631         [INSTR_REGPREFETCH_RM] = instr_reg_export,
11632         [INSTR_REGPREFETCH_RI] = instr_reg_export,
11633
11634         [INSTR_REGRD_HRH] = instr_reg_export,
11635         [INSTR_REGRD_HRM] = instr_reg_export,
11636         [INSTR_REGRD_MRH] = instr_reg_export,
11637         [INSTR_REGRD_MRM] = instr_reg_export,
11638         [INSTR_REGRD_HRI] = instr_reg_export,
11639         [INSTR_REGRD_MRI] = instr_reg_export,
11640
11641         [INSTR_REGWR_RHH] = instr_reg_export,
11642         [INSTR_REGWR_RHM] = instr_reg_export,
11643         [INSTR_REGWR_RMH] = instr_reg_export,
11644         [INSTR_REGWR_RMM] = instr_reg_export,
11645         [INSTR_REGWR_RHI] = instr_reg_export,
11646         [INSTR_REGWR_RMI] = instr_reg_export,
11647         [INSTR_REGWR_RIH] = instr_reg_export,
11648         [INSTR_REGWR_RIM] = instr_reg_export,
11649         [INSTR_REGWR_RII] = instr_reg_export,
11650
11651         [INSTR_REGADD_RHH] = instr_reg_export,
11652         [INSTR_REGADD_RHM] = instr_reg_export,
11653         [INSTR_REGADD_RMH] = instr_reg_export,
11654         [INSTR_REGADD_RMM] = instr_reg_export,
11655         [INSTR_REGADD_RHI] = instr_reg_export,
11656         [INSTR_REGADD_RMI] = instr_reg_export,
11657         [INSTR_REGADD_RIH] = instr_reg_export,
11658         [INSTR_REGADD_RIM] = instr_reg_export,
11659         [INSTR_REGADD_RII] = instr_reg_export,
11660
11661         [INSTR_METPREFETCH_H] = instr_meter_export,
11662         [INSTR_METPREFETCH_M] = instr_meter_export,
11663         [INSTR_METPREFETCH_I] = instr_meter_export,
11664
11665         [INSTR_METER_HHM] = instr_meter_export,
11666         [INSTR_METER_HHI] = instr_meter_export,
11667         [INSTR_METER_HMM] = instr_meter_export,
11668         [INSTR_METER_HMI] = instr_meter_export,
11669         [INSTR_METER_MHM] = instr_meter_export,
11670         [INSTR_METER_MHI] = instr_meter_export,
11671         [INSTR_METER_MMM] = instr_meter_export,
11672         [INSTR_METER_MMI] = instr_meter_export,
11673         [INSTR_METER_IHM] = instr_meter_export,
11674         [INSTR_METER_IHI] = instr_meter_export,
11675         [INSTR_METER_IMM] = instr_meter_export,
11676         [INSTR_METER_IMI] = instr_meter_export,
11677
11678         [INSTR_TABLE] = instr_table_export,
11679         [INSTR_TABLE_AF] = instr_table_export,
11680         [INSTR_SELECTOR] = instr_table_export,
11681         [INSTR_LEARNER] = instr_table_export,
11682         [INSTR_LEARNER_AF] = instr_table_export,
11683
11684         [INSTR_LEARNER_LEARN] = instr_learn_export,
11685         [INSTR_LEARNER_REARM] = instr_rearm_export,
11686         [INSTR_LEARNER_REARM_NEW] = instr_rearm_export,
11687         [INSTR_LEARNER_FORGET] = instr_forget_export,
11688
11689         [INSTR_EXTERN_OBJ] = instr_extern_export,
11690         [INSTR_EXTERN_FUNC] = instr_extern_export,
11691
11692         [INSTR_JMP] = instr_jmp_export,
11693         [INSTR_JMP_VALID] = instr_jmp_export,
11694         [INSTR_JMP_INVALID] = instr_jmp_export,
11695         [INSTR_JMP_HIT] = instr_jmp_export,
11696         [INSTR_JMP_MISS] = instr_jmp_export,
11697         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11698         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11699
11700         [INSTR_JMP_EQ] = instr_jmp_export,
11701         [INSTR_JMP_EQ_MH] = instr_jmp_export,
11702         [INSTR_JMP_EQ_HM] = instr_jmp_export,
11703         [INSTR_JMP_EQ_HH] = instr_jmp_export,
11704         [INSTR_JMP_EQ_I] = instr_jmp_export,
11705
11706         [INSTR_JMP_NEQ] = instr_jmp_export,
11707         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11708         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11709         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11710         [INSTR_JMP_NEQ_I] = instr_jmp_export,
11711
11712         [INSTR_JMP_LT] = instr_jmp_export,
11713         [INSTR_JMP_LT_MH] = instr_jmp_export,
11714         [INSTR_JMP_LT_HM] = instr_jmp_export,
11715         [INSTR_JMP_LT_HH] = instr_jmp_export,
11716         [INSTR_JMP_LT_MI] = instr_jmp_export,
11717         [INSTR_JMP_LT_HI] = instr_jmp_export,
11718
11719         [INSTR_JMP_GT] = instr_jmp_export,
11720         [INSTR_JMP_GT_MH] = instr_jmp_export,
11721         [INSTR_JMP_GT_HM] = instr_jmp_export,
11722         [INSTR_JMP_GT_HH] = instr_jmp_export,
11723         [INSTR_JMP_GT_MI] = instr_jmp_export,
11724         [INSTR_JMP_GT_HI] = instr_jmp_export,
11725
11726         [INSTR_RETURN] = instr_return_export,
11727 };
11728
11729 static void
11730 action_data_codegen(struct action *a, FILE *f)
11731 {
11732         uint32_t i;
11733
11734         fprintf(f,
11735                 "static const struct instruction action_%s_instructions[] = {\n",
11736                 a->name);
11737
11738         for (i = 0; i < a->n_instructions; i++) {
11739                 struct instruction *instr = &a->instructions[i];
11740                 instruction_export_t func = export_table[instr->type];
11741
11742                 func(instr, f);
11743         }
11744
11745         fprintf(f, "};\n");
11746 }
11747
11748 static const char *
11749 instr_type_to_func(struct instruction *instr)
11750 {
11751         switch (instr->type) {
11752         case INSTR_RX: return NULL;
11753
11754         case INSTR_TX: return "__instr_tx_exec";
11755         case INSTR_TX_I: return "__instr_tx_i_exec";
11756         case INSTR_DROP: return "__instr_drop_exec";
11757         case INSTR_MIRROR: return "__instr_mirror_exec";
11758         case INSTR_RECIRCULATE: return "__instr_recirculate_exec";
11759         case INSTR_RECIRCID: return "__instr_recircid_exec";
11760
11761         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11762         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11763         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11764         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11765         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11766         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11767         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11768         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11769
11770         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11771
11772         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11773
11774         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11775         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11776         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11777         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11778         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11779         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11780         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11781         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11782         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11783
11784         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11785         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11786
11787         case INSTR_MOV: return "__instr_mov_exec";
11788         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11789         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11790         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11791         case INSTR_MOV_I: return "__instr_mov_i_exec";
11792
11793         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11794         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11795         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11796         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11797         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11798         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11799         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11800         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11801
11802         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11803         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11804         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11805         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11806         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11807         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11808
11809         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11810         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11811         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11812         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11813         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11814         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11815
11816         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11817         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11818         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11819         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11820
11821         case INSTR_ALU_AND: return "__instr_alu_and_exec";
11822         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11823         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11824         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11825         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11826
11827         case INSTR_ALU_OR: return "__instr_alu_or_exec";
11828         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11829         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11830         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11831         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11832
11833         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11834         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11835         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11836         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11837         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11838
11839         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11840         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11841         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11842         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11843         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11844         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11845
11846         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11847         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11848         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11849         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11850         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11851         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11852
11853         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11854         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11855         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11856
11857         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11858         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11859         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11860         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11861         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11862         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11863
11864         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11865         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11866         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11867         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11868         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11869         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11870         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11871         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11872         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11873
11874         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11875         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11876         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11877         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11878         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11879         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11880         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11881         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11882         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11883
11884         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11885         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11886         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11887
11888         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11889         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11890         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11891         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11892         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11893         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11894         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11895         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11896         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11897         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11898         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11899         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11900
11901         case INSTR_TABLE: return NULL;
11902         case INSTR_TABLE_AF: return NULL;
11903         case INSTR_SELECTOR: return NULL;
11904         case INSTR_LEARNER: return NULL;
11905         case INSTR_LEARNER_AF: return NULL;
11906
11907         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11908         case INSTR_LEARNER_REARM: return "__instr_rearm_exec";
11909         case INSTR_LEARNER_REARM_NEW: return "__instr_rearm_new_exec";
11910         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11911
11912         case INSTR_EXTERN_OBJ: return NULL;
11913         case INSTR_EXTERN_FUNC: return NULL;
11914
11915         case INSTR_JMP: return NULL;
11916         case INSTR_JMP_VALID: return NULL;
11917         case INSTR_JMP_INVALID: return NULL;
11918         case INSTR_JMP_HIT: return NULL;
11919         case INSTR_JMP_MISS: return NULL;
11920         case INSTR_JMP_ACTION_HIT: return NULL;
11921         case INSTR_JMP_ACTION_MISS: return NULL;
11922         case INSTR_JMP_EQ: return NULL;
11923         case INSTR_JMP_EQ_MH: return NULL;
11924         case INSTR_JMP_EQ_HM: return NULL;
11925         case INSTR_JMP_EQ_HH: return NULL;
11926         case INSTR_JMP_EQ_I: return NULL;
11927         case INSTR_JMP_NEQ: return NULL;
11928         case INSTR_JMP_NEQ_MH: return NULL;
11929         case INSTR_JMP_NEQ_HM: return NULL;
11930         case INSTR_JMP_NEQ_HH: return NULL;
11931         case INSTR_JMP_NEQ_I: return NULL;
11932         case INSTR_JMP_LT: return NULL;
11933         case INSTR_JMP_LT_MH: return NULL;
11934         case INSTR_JMP_LT_HM: return NULL;
11935         case INSTR_JMP_LT_HH: return NULL;
11936         case INSTR_JMP_LT_MI: return NULL;
11937         case INSTR_JMP_LT_HI: return NULL;
11938         case INSTR_JMP_GT: return NULL;
11939         case INSTR_JMP_GT_MH: return NULL;
11940         case INSTR_JMP_GT_HM: return NULL;
11941         case INSTR_JMP_GT_HH: return NULL;
11942         case INSTR_JMP_GT_MI: return NULL;
11943         case INSTR_JMP_GT_HI: return NULL;
11944
11945         case INSTR_RETURN: return NULL;
11946
11947         default: return NULL;
11948         }
11949 }
11950
11951 static void
11952 action_instr_does_tx_codegen(struct action *a,
11953                         uint32_t instr_pos,
11954                         struct instruction *instr,
11955                         FILE *f)
11956 {
11957         fprintf(f,
11958                 "%s(p, t, &action_%s_instructions[%u]);\n"
11959                 "\tthread_ip_reset(p, t);\n"
11960                 "\tinstr_rx_exec(p);\n"
11961                 "\treturn;\n",
11962                 instr_type_to_func(instr),
11963                 a->name,
11964                 instr_pos);
11965 }
11966
11967 static void
11968 action_instr_extern_obj_codegen(struct action *a,
11969                                 uint32_t instr_pos,
11970                                 FILE *f)
11971 {
11972         fprintf(f,
11973                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11974                 a->name,
11975                 instr_pos);
11976 }
11977
11978 static void
11979 action_instr_extern_func_codegen(struct action *a,
11980                                  uint32_t instr_pos,
11981                                  FILE *f)
11982 {
11983         fprintf(f,
11984                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11985                 a->name,
11986                 instr_pos);
11987 }
11988
11989 static void
11990 action_instr_jmp_codegen(struct action *a,
11991                          uint32_t instr_pos,
11992                          struct instruction *instr,
11993                          struct instruction_data *data,
11994                          FILE *f)
11995 {
11996         switch (instr->type) {
11997         case INSTR_JMP:
11998                 fprintf(f,
11999                         "goto %s;\n",
12000                         data->jmp_label);
12001                 return;
12002
12003         case INSTR_JMP_VALID:
12004                 fprintf(f,
12005                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
12006                         "\t\tgoto %s;\n",
12007                         a->name,
12008                         instr_pos,
12009                         data->jmp_label);
12010                 return;
12011
12012         case INSTR_JMP_INVALID:
12013                 fprintf(f,
12014                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
12015                         "\t\tgoto %s;\n",
12016                         a->name,
12017                         instr_pos,
12018                         data->jmp_label);
12019                 return;
12020
12021         case INSTR_JMP_HIT:
12022                 fprintf(f,
12023                         "if (t->hit)\n"
12024                         "\t\tgoto %s;\n",
12025                         data->jmp_label);
12026                 return;
12027
12028         case INSTR_JMP_MISS:
12029                 fprintf(f,
12030                         "if (!t->hit)\n"
12031                         "\t\tgoto %s;\n",
12032                         data->jmp_label);
12033                 return;
12034
12035         case INSTR_JMP_ACTION_HIT:
12036                 fprintf(f,
12037                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
12038                         "\t\tgoto %s;\n",
12039                         a->name,
12040                         instr_pos,
12041                         data->jmp_label);
12042                 return;
12043
12044         case INSTR_JMP_ACTION_MISS:
12045                 fprintf(f,
12046                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
12047                         "\t\tgoto %s;\n",
12048                         a->name,
12049                         instr_pos,
12050                         data->jmp_label);
12051                 return;
12052
12053         case INSTR_JMP_EQ:
12054                 fprintf(f,
12055                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12056                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12057                         "\t\tgoto %s;\n",
12058                         a->name,
12059                         instr_pos,
12060                         a->name,
12061                         instr_pos,
12062                         data->jmp_label);
12063                 return;
12064
12065         case INSTR_JMP_EQ_MH:
12066                 fprintf(f,
12067                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12068                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12069                         "\t\tgoto %s;\n",
12070                         a->name,
12071                         instr_pos,
12072                         a->name,
12073                         instr_pos,
12074                         data->jmp_label);
12075                 return;
12076
12077         case INSTR_JMP_EQ_HM:
12078                 fprintf(f,
12079                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
12080                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12081                         "\t\tgoto %s;\n",
12082                         a->name,
12083                         instr_pos,
12084                         a->name,
12085                         instr_pos,
12086                         data->jmp_label);
12087                 return;
12088
12089         case INSTR_JMP_EQ_HH:
12090                 fprintf(f,
12091                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
12092                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12093                         "\t\tgoto %s;\n",
12094                         a->name,
12095                         instr_pos,
12096                         a->name,
12097                         instr_pos,
12098                         data->jmp_label);
12099                 return;
12100
12101         case INSTR_JMP_EQ_I:
12102                 fprintf(f,
12103                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12104                         "action_%s_instructions[%u].jmp.b_val)\n"
12105                         "\t\tgoto %s;\n",
12106                         a->name,
12107                         instr_pos,
12108                         a->name,
12109                         instr_pos,
12110                         data->jmp_label);
12111                 return;
12112
12113         case INSTR_JMP_NEQ:
12114                 fprintf(f,
12115                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12116                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12117                         "\t\tgoto %s;\n",
12118                         a->name,
12119                         instr_pos,
12120                         a->name,
12121                         instr_pos,
12122                         data->jmp_label);
12123                 return;
12124
12125         case INSTR_JMP_NEQ_MH:
12126                 fprintf(f,
12127                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12128                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12129                         "\t\tgoto %s;\n",
12130                         a->name,
12131                         instr_pos,
12132                         a->name,
12133                         instr_pos,
12134                         data->jmp_label);
12135                 return;
12136
12137         case INSTR_JMP_NEQ_HM:
12138                 fprintf(f,
12139                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
12140                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12141                         "\t\tgoto %s;\n",
12142                         a->name,
12143                         instr_pos,
12144                         a->name,
12145                         instr_pos,
12146                         data->jmp_label);
12147                 return;
12148
12149         case INSTR_JMP_NEQ_HH:
12150                 fprintf(f,
12151                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
12152                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12153                         "\t\tgoto %s;\n",
12154                         a->name,
12155                         instr_pos,
12156                         a->name,
12157                         instr_pos,
12158                         data->jmp_label);
12159                 return;
12160
12161         case INSTR_JMP_NEQ_I:
12162                 fprintf(f,
12163                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12164                         "action_%s_instructions[%u].jmp.b_val)\n"
12165                         "\t\tgoto %s;\n",
12166                         a->name,
12167                         instr_pos,
12168                         a->name,
12169                         instr_pos,
12170                         data->jmp_label);
12171                 return;
12172
12173         case INSTR_JMP_LT:
12174                 fprintf(f,
12175                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12176                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12177                         "\t\tgoto %s;\n",
12178                         a->name,
12179                         instr_pos,
12180                         a->name,
12181                         instr_pos,
12182                         data->jmp_label);
12183                 return;
12184
12185         case INSTR_JMP_LT_MH:
12186                 fprintf(f,
12187                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12188                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12189                         "\t\tgoto %s;\n",
12190                         a->name,
12191                         instr_pos,
12192                         a->name,
12193                         instr_pos,
12194                         data->jmp_label);
12195                 return;
12196
12197         case INSTR_JMP_LT_HM:
12198                 fprintf(f,
12199                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12200                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12201                         "\t\tgoto %s;\n",
12202                         a->name,
12203                         instr_pos,
12204                         a->name,
12205                         instr_pos,
12206                         data->jmp_label);
12207                 return;
12208
12209         case INSTR_JMP_LT_HH:
12210                 fprintf(f,
12211                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12212                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12213                         "\t\tgoto %s;\n",
12214                         a->name,
12215                         instr_pos,
12216                         a->name,
12217                         instr_pos,
12218                         data->jmp_label);
12219                 return;
12220
12221         case INSTR_JMP_LT_MI:
12222                 fprintf(f,
12223                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12224                         "action_%s_instructions[%u].jmp.b_val)\n"
12225                         "\t\tgoto %s;\n",
12226                         a->name,
12227                         instr_pos,
12228                         a->name,
12229                         instr_pos,
12230                         data->jmp_label);
12231                 return;
12232
12233         case INSTR_JMP_LT_HI:
12234                 fprintf(f,
12235                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12236                         "action_%s_instructions[%u].jmp.b_val)\n"
12237                         "\t\tgoto %s;\n",
12238                         a->name,
12239                         instr_pos,
12240                         a->name,
12241                         instr_pos,
12242                         data->jmp_label);
12243                 return;
12244
12245         case INSTR_JMP_GT:
12246                 fprintf(f,
12247                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12248                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12249                         "\t\tgoto %s;\n",
12250                         a->name,
12251                         instr_pos,
12252                         a->name,
12253                         instr_pos,
12254                         data->jmp_label);
12255                 return;
12256
12257         case INSTR_JMP_GT_MH:
12258                 fprintf(f,
12259                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12260                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12261                         "\t\tgoto %s;\n",
12262                         a->name,
12263                         instr_pos,
12264                         a->name,
12265                         instr_pos,
12266                         data->jmp_label);
12267                 return;
12268
12269         case INSTR_JMP_GT_HM:
12270                 fprintf(f,
12271                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12272                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12273                         "\t\tgoto %s;\n",
12274                         a->name,
12275                         instr_pos,
12276                         a->name,
12277                         instr_pos,
12278                         data->jmp_label);
12279                 return;
12280
12281         case INSTR_JMP_GT_HH:
12282                 fprintf(f,
12283                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12284                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12285                         "\t\tgoto %s;\n",
12286                         a->name,
12287                         instr_pos,
12288                         a->name,
12289                         instr_pos,
12290                         data->jmp_label);
12291                 return;
12292
12293         case INSTR_JMP_GT_MI:
12294                 fprintf(f,
12295                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12296                         "action_%s_instructions[%u].jmp.b_val)\n"
12297                         "\t\tgoto %s;\n",
12298                         a->name,
12299                         instr_pos,
12300                         a->name,
12301                         instr_pos,
12302                         data->jmp_label);
12303                 return;
12304
12305         case INSTR_JMP_GT_HI:
12306                 fprintf(f,
12307                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12308                         "action_%s_instructions[%u].jmp.b_val)\n"
12309                         "\t\tgoto %s;\n",
12310                         a->name,
12311                         instr_pos,
12312                         a->name,
12313                         instr_pos,
12314                         data->jmp_label);
12315                 return;
12316
12317         default:
12318                 return;
12319         }
12320 }
12321
12322 static void
12323 action_instr_return_codegen(FILE *f)
12324 {
12325         fprintf(f,
12326                 "return;\n");
12327 }
12328
12329 static void
12330 action_instr_codegen(struct action *a, FILE *f)
12331 {
12332         uint32_t i;
12333
12334         fprintf(f,
12335                 "void\n"
12336                 "action_%s_run(struct rte_swx_pipeline *p)\n"
12337                 "{\n"
12338                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12339                 "\n",
12340                 a->name);
12341
12342         for (i = 0; i < a->n_instructions; i++) {
12343                 struct instruction *instr = &a->instructions[i];
12344                 struct instruction_data *data = &a->instruction_data[i];
12345
12346                 /* Label, if present. */
12347                 if (data->label[0])
12348                         fprintf(f, "\n%s : ", data->label);
12349                 else
12350                         fprintf(f, "\n\t");
12351
12352                 /* TX instruction type. */
12353                 if (instruction_does_tx(instr)) {
12354                         action_instr_does_tx_codegen(a, i, instr, f);
12355                         continue;
12356                 }
12357
12358                 /* Extern object/function instruction type. */
12359                 if (instr->type == INSTR_EXTERN_OBJ) {
12360                         action_instr_extern_obj_codegen(a, i, f);
12361                         continue;
12362                 }
12363
12364                 if (instr->type == INSTR_EXTERN_FUNC) {
12365                         action_instr_extern_func_codegen(a, i, f);
12366                         continue;
12367                 }
12368
12369                 /* Jump instruction type. */
12370                 if (instruction_is_jmp(instr)) {
12371                         action_instr_jmp_codegen(a, i, instr, data, f);
12372                         continue;
12373                 }
12374
12375                 /* Return instruction type. */
12376                 if (instr->type == INSTR_RETURN) {
12377                         action_instr_return_codegen(f);
12378                         continue;
12379                 }
12380
12381                 /* Any other instruction type. */
12382                 fprintf(f,
12383                         "%s(p, t, &action_%s_instructions[%u]);\n",
12384                         instr_type_to_func(instr),
12385                         a->name,
12386                         i);
12387         }
12388
12389         fprintf(f, "}\n\n");
12390 }
12391
12392 struct instruction_group {
12393         TAILQ_ENTRY(instruction_group) node;
12394
12395         uint32_t group_id;
12396
12397         uint32_t first_instr_id;
12398
12399         uint32_t last_instr_id;
12400
12401         instr_exec_t func;
12402 };
12403
12404 TAILQ_HEAD(instruction_group_list, instruction_group);
12405
12406 static struct instruction_group *
12407 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
12408 {
12409         struct instruction_group *g;
12410
12411         TAILQ_FOREACH(g, igl, node)
12412                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
12413                         return g;
12414
12415         return NULL;
12416 }
12417
12418 static void
12419 instruction_group_list_free(struct instruction_group_list *igl)
12420 {
12421         if (!igl)
12422                 return;
12423
12424         for ( ; ; ) {
12425                 struct instruction_group *g;
12426
12427                 g = TAILQ_FIRST(igl);
12428                 if (!g)
12429                         break;
12430
12431                 TAILQ_REMOVE(igl, g, node);
12432                 free(g);
12433         }
12434
12435         free(igl);
12436 }
12437
12438 static struct instruction_group_list *
12439 instruction_group_list_create(struct rte_swx_pipeline *p)
12440 {
12441         struct instruction_group_list *igl = NULL;
12442         struct instruction_group *g = NULL;
12443         uint32_t n_groups = 0, i;
12444
12445         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
12446                 goto error;
12447
12448         /* List init. */
12449         igl = calloc(1, sizeof(struct instruction_group_list));
12450         if (!igl)
12451                 goto error;
12452
12453         TAILQ_INIT(igl);
12454
12455         /* Allocate the first group. */
12456         g = calloc(1, sizeof(struct instruction_group));
12457         if (!g)
12458                 goto error;
12459
12460         /* Iteration 1: Separate the instructions into groups based on the thread yield
12461          * instructions. Do not worry about the jump instructions at this point.
12462          */
12463         for (i = 0; i < p->n_instructions; i++) {
12464                 struct instruction *instr = &p->instructions[i];
12465
12466                 /* Check for thread yield instructions. */
12467                 if (!instruction_does_thread_yield(instr))
12468                         continue;
12469
12470                 /* If the current group contains at least one instruction, then finalize it (with
12471                  * the previous instruction), add it to the list and allocate a new group (that
12472                  * starts with the current instruction).
12473                  */
12474                 if (i - g->first_instr_id) {
12475                         /* Finalize the group. */
12476                         g->last_instr_id = i - 1;
12477
12478                         /* Add the group to the list. Advance the number of groups. */
12479                         TAILQ_INSERT_TAIL(igl, g, node);
12480                         n_groups++;
12481
12482                         /* Allocate a new group. */
12483                         g = calloc(1, sizeof(struct instruction_group));
12484                         if (!g)
12485                                 goto error;
12486
12487                         /* Initialize the new group. */
12488                         g->group_id = n_groups;
12489                         g->first_instr_id = i;
12490                 }
12491
12492                 /* Finalize the current group (with the current instruction, therefore this group
12493                  * contains just the current thread yield instruction), add it to the list and
12494                  * allocate a new group (that starts with the next instruction).
12495                  */
12496
12497                 /* Finalize the group. */
12498                 g->last_instr_id = i;
12499
12500                 /* Add the group to the list. Advance the number of groups. */
12501                 TAILQ_INSERT_TAIL(igl, g, node);
12502                 n_groups++;
12503
12504                 /* Allocate a new group. */
12505                 g = calloc(1, sizeof(struct instruction_group));
12506                 if (!g)
12507                         goto error;
12508
12509                 /* Initialize the new group. */
12510                 g->group_id = n_groups;
12511                 g->first_instr_id = i + 1;
12512         }
12513
12514         /* Handle the last group. */
12515         if (i - g->first_instr_id) {
12516                 /* Finalize the group. */
12517                 g->last_instr_id = i - 1;
12518
12519                 /* Add the group to the list. Advance the number of groups. */
12520                 TAILQ_INSERT_TAIL(igl, g, node);
12521                 n_groups++;
12522         } else
12523                 free(g);
12524
12525         g = NULL;
12526
12527         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
12528          * the destination of a jump instruction located in a different group ("far jump"), then the
12529          * current group has to be split, so that the instruction representing the far jump
12530          * destination is at the start of its group.
12531          */
12532         for ( ; ; ) {
12533                 int is_modified = 0;
12534
12535                 for (i = 0; i < p->n_instructions; i++) {
12536                         struct instruction_data *data = &p->instruction_data[i];
12537                         struct instruction_group *g;
12538                         uint32_t j;
12539
12540                         /* Continue when the current instruction is not a jump destination. */
12541                         if (!data->n_users)
12542                                 continue;
12543
12544                         g = instruction_group_list_group_find(igl, i);
12545                         if (!g)
12546                                 goto error;
12547
12548                         /* Find out all the jump instructions with this destination. */
12549                         for (j = 0; j < p->n_instructions; j++) {
12550                                 struct instruction *jmp_instr = &p->instructions[j];
12551                                 struct instruction_data *jmp_data = &p->instruction_data[j];
12552                                 struct instruction_group *jmp_g, *new_g;
12553
12554                                 /* Continue when not a jump instruction. Even when jump instruction,
12555                                  * continue when the jump destination is not this instruction.
12556                                  */
12557                                 if (!instruction_is_jmp(jmp_instr) ||
12558                                     strcmp(jmp_data->jmp_label, data->label))
12559                                         continue;
12560
12561                                 jmp_g = instruction_group_list_group_find(igl, j);
12562                                 if (!jmp_g)
12563                                         goto error;
12564
12565                                 /* Continue when both the jump instruction and the jump destination
12566                                  * instruction are in the same group. Even when in different groups,
12567                                  * still continue if the jump destination instruction is already the
12568                                  * first instruction of its group.
12569                                  */
12570                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
12571                                         continue;
12572
12573                                 /* Split the group of the current jump destination instruction to
12574                                  * make this instruction the first instruction of a new group.
12575                                  */
12576                                 new_g = calloc(1, sizeof(struct instruction_group));
12577                                 if (!new_g)
12578                                         goto error;
12579
12580                                 new_g->group_id = n_groups;
12581                                 new_g->first_instr_id = i;
12582                                 new_g->last_instr_id = g->last_instr_id;
12583
12584                                 g->last_instr_id = i - 1;
12585
12586                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
12587                                 n_groups++;
12588                                 is_modified = 1;
12589
12590                                 /* The decision to split this group (to make the current instruction
12591                                  * the first instruction of a new group) is already taken and fully
12592                                  * implemented, so no need to search for more reasons to do it.
12593                                  */
12594                                 break;
12595                         }
12596                 }
12597
12598                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
12599                  * previously considered local (i.e. the jump destination is in the same group as
12600                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
12601                  * different group than the jump instruction). Wost case scenario: each instruction
12602                  * that is a jump destination ends up as the first instruction of its group.
12603                  */
12604                 if (!is_modified)
12605                         break;
12606         }
12607
12608         /* Re-assign the group IDs to be in incremental order. */
12609         i = 0;
12610         TAILQ_FOREACH(g, igl, node) {
12611                 g->group_id = i;
12612
12613                 i++;
12614         }
12615
12616         return igl;
12617
12618 error:
12619         instruction_group_list_free(igl);
12620
12621         free(g);
12622
12623         return NULL;
12624 }
12625
12626 static void
12627 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12628                                uint32_t instr_pos,
12629                                struct instruction *instr,
12630                                FILE *f)
12631 {
12632         fprintf(f,
12633                 "%s(p, t, &pipeline_instructions[%u]);\n"
12634                 "\tthread_ip_reset(p, t);\n"
12635                 "\tinstr_rx_exec(p);\n"
12636                 "\treturn;\n",
12637                 instr_type_to_func(instr),
12638                 instr_pos);
12639 }
12640
12641 static int
12642 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12643                            struct instruction_group_list *igl,
12644                            uint32_t jmp_instr_id,
12645                            struct instruction *jmp_instr,
12646                            struct instruction_data *jmp_data,
12647                            FILE *f)
12648 {
12649         struct instruction_group *jmp_g, *g;
12650         struct instruction_data *data;
12651         uint32_t instr_id;
12652
12653         switch (jmp_instr->type) {
12654         case INSTR_JMP:
12655                 break;
12656
12657         case INSTR_JMP_VALID:
12658                 fprintf(f,
12659                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12660                         jmp_instr_id);
12661                 break;
12662
12663         case INSTR_JMP_INVALID:
12664                 fprintf(f,
12665                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12666                         jmp_instr_id);
12667                 break;
12668
12669         case INSTR_JMP_HIT:
12670                 fprintf(f,
12671                         "if (t->hit)\n");
12672                 break;
12673
12674         case INSTR_JMP_MISS:
12675                 fprintf(f,
12676                         "if (!t->hit)\n");
12677                 break;
12678
12679         case INSTR_JMP_ACTION_HIT:
12680                 fprintf(f,
12681                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12682                         jmp_instr_id);
12683                 break;
12684
12685         case INSTR_JMP_ACTION_MISS:
12686                 fprintf(f,
12687                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12688                         jmp_instr_id);
12689                 break;
12690
12691         case INSTR_JMP_EQ:
12692                 fprintf(f,
12693                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12694                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12695                         jmp_instr_id,
12696                         jmp_instr_id);
12697                 break;
12698
12699         case INSTR_JMP_EQ_MH:
12700                 fprintf(f,
12701                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12702                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12703                         jmp_instr_id,
12704                         jmp_instr_id);
12705                 break;
12706
12707         case INSTR_JMP_EQ_HM:
12708                 fprintf(f,
12709                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12710                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12711                         jmp_instr_id,
12712                         jmp_instr_id);
12713                 break;
12714
12715         case INSTR_JMP_EQ_HH:
12716                 fprintf(f,
12717                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12718                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12719                         jmp_instr_id,
12720                         jmp_instr_id);
12721                 break;
12722
12723         case INSTR_JMP_EQ_I:
12724                 fprintf(f,
12725                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12726                         "pipeline_instructions[%u].jmp.b_val)",
12727                         jmp_instr_id,
12728                         jmp_instr_id);
12729                 break;
12730
12731         case INSTR_JMP_NEQ:
12732                 fprintf(f,
12733                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12734                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12735                         jmp_instr_id,
12736                         jmp_instr_id);
12737                 break;
12738
12739         case INSTR_JMP_NEQ_MH:
12740                 fprintf(f,
12741                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12742                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12743                         jmp_instr_id,
12744                         jmp_instr_id);
12745                 break;
12746
12747         case INSTR_JMP_NEQ_HM:
12748                 fprintf(f,
12749                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12750                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12751                         jmp_instr_id,
12752                         jmp_instr_id);
12753                 break;
12754
12755         case INSTR_JMP_NEQ_HH:
12756                 fprintf(f,
12757                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12758                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12759                         jmp_instr_id,
12760                         jmp_instr_id);
12761                 break;
12762
12763         case INSTR_JMP_NEQ_I:
12764                 fprintf(f,
12765                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12766                         "pipeline_instructions[%u].jmp.b_val)",
12767                         jmp_instr_id,
12768                         jmp_instr_id);
12769                 break;
12770
12771         case INSTR_JMP_LT:
12772                 fprintf(f,
12773                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12774                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12775                         jmp_instr_id,
12776                         jmp_instr_id);
12777                 break;
12778
12779         case INSTR_JMP_LT_MH:
12780                 fprintf(f,
12781                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12782                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12783                         jmp_instr_id,
12784                         jmp_instr_id);
12785                 break;
12786
12787         case INSTR_JMP_LT_HM:
12788                 fprintf(f,
12789                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12790                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12791                         jmp_instr_id,
12792                         jmp_instr_id);
12793                 break;
12794
12795         case INSTR_JMP_LT_HH:
12796                 fprintf(f,
12797                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12798                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12799                         jmp_instr_id,
12800                         jmp_instr_id);
12801                 break;
12802
12803         case INSTR_JMP_LT_MI:
12804                 fprintf(f,
12805                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12806                         "pipeline_instructions[%u].jmp.b_val)",
12807                         jmp_instr_id,
12808                         jmp_instr_id);
12809                 break;
12810
12811         case INSTR_JMP_LT_HI:
12812                 fprintf(f,
12813                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12814                         "pipeline_instructions[%u].jmp.b_val)",
12815                         jmp_instr_id,
12816                         jmp_instr_id);
12817                 break;
12818
12819         case INSTR_JMP_GT:
12820                 fprintf(f,
12821                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12822                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12823                         jmp_instr_id,
12824                         jmp_instr_id);
12825                 break;
12826
12827         case INSTR_JMP_GT_MH:
12828                 fprintf(f,
12829                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12830                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12831                         jmp_instr_id,
12832                         jmp_instr_id);
12833                 break;
12834
12835         case INSTR_JMP_GT_HM:
12836                 fprintf(f,
12837                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12838                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12839                         jmp_instr_id,
12840                         jmp_instr_id);
12841                 break;
12842
12843         case INSTR_JMP_GT_HH:
12844                 fprintf(f,
12845                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12846                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12847                         jmp_instr_id,
12848                         jmp_instr_id);
12849                 break;
12850
12851         case INSTR_JMP_GT_MI:
12852                 fprintf(f,
12853                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12854                         "pipeline_instructions[%u].jmp.b_val)",
12855                         jmp_instr_id,
12856                         jmp_instr_id);
12857                 break;
12858
12859         case INSTR_JMP_GT_HI:
12860                 fprintf(f,
12861                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12862                         "pipeline_instructions[%u].jmp.b_val)",
12863                         jmp_instr_id,
12864                         jmp_instr_id);
12865                 break;
12866
12867         default:
12868                 break;
12869         }
12870
12871         /* Find the instruction group of the jump instruction. */
12872         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12873         if (!jmp_g)
12874                 return -EINVAL;
12875
12876         /* Find the instruction group of the jump destination instruction. */
12877         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12878         if (!data)
12879                 return -EINVAL;
12880
12881         instr_id = data - p->instruction_data;
12882
12883         g = instruction_group_list_group_find(igl, instr_id);
12884         if (!g)
12885                 return -EINVAL;
12886
12887         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12888          * instruction group).
12889          */
12890         if (g->group_id == jmp_g->group_id)
12891                 fprintf(f,
12892                         "\n\t\tgoto %s;\n",
12893                         jmp_data->jmp_label);
12894         else
12895                 fprintf(f,
12896                         " {\n"
12897                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12898                         "\t\treturn;\n"
12899                         "\t}\n\n",
12900                         g->group_id);
12901
12902         return 0;
12903 }
12904
12905 static void
12906 instruction_group_list_codegen(struct instruction_group_list *igl,
12907                                struct rte_swx_pipeline *p,
12908                                FILE *f)
12909 {
12910         struct instruction_group *g;
12911         uint32_t i;
12912         int is_required = 0;
12913
12914         /* Check if code generation is required. */
12915         TAILQ_FOREACH(g, igl, node)
12916                 if (g->first_instr_id < g->last_instr_id)
12917                         is_required = 1;
12918
12919         if (!is_required)
12920                 return;
12921
12922         /* Generate the code for the pipeline instruction array. */
12923         fprintf(f,
12924                 "static const struct instruction pipeline_instructions[] = {\n");
12925
12926         for (i = 0; i < p->n_instructions; i++) {
12927                 struct instruction *instr = &p->instructions[i];
12928                 instruction_export_t func = export_table[instr->type];
12929
12930                 func(instr, f);
12931         }
12932
12933         fprintf(f, "};\n\n");
12934
12935         /* Generate the code for the pipeline functions: one function for each instruction group
12936          * that contains more than one instruction.
12937          */
12938         TAILQ_FOREACH(g, igl, node) {
12939                 struct instruction *last_instr;
12940                 uint32_t j;
12941
12942                 /* Skip if group contains a single instruction. */
12943                 if (g->last_instr_id == g->first_instr_id)
12944                         continue;
12945
12946                 /* Generate new pipeline function. */
12947                 fprintf(f,
12948                         "void\n"
12949                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12950                         "{\n"
12951                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12952                         "\n",
12953                         g->group_id);
12954
12955                 /* Generate the code for each pipeline instruction. */
12956                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12957                         struct instruction *instr = &p->instructions[j];
12958                         struct instruction_data *data = &p->instruction_data[j];
12959
12960                         /* Label, if present. */
12961                         if (data->label[0])
12962                                 fprintf(f, "\n%s : ", data->label);
12963                         else
12964                                 fprintf(f, "\n\t");
12965
12966                         /* TX instruction type. */
12967                         if (instruction_does_tx(instr)) {
12968                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12969                                 continue;
12970                         }
12971
12972                         /* Jump instruction type. */
12973                         if (instruction_is_jmp(instr)) {
12974                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12975                                 continue;
12976                         }
12977
12978                         /* Any other instruction type. */
12979                         fprintf(f,
12980                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12981                                 instr_type_to_func(instr),
12982                                 j);
12983                 }
12984
12985                 /* Finalize the generated pipeline function. For some instructions such as TX,
12986                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12987                  * decided unconditionally and the instruction pointer of the current thread set
12988                  * accordingly; for all the other instructions, the instruction pointer must be
12989                  * incremented now.
12990                  */
12991                 last_instr = &p->instructions[g->last_instr_id];
12992
12993                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12994                         fprintf(f,
12995                                 "thread_ip_inc(p);\n");
12996
12997                 fprintf(f,
12998                         "}\n"
12999                         "\n");
13000         }
13001 }
13002
13003 static uint32_t
13004 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
13005 {
13006         struct instruction_group *g;
13007         uint32_t n_custom_instr = 0;
13008
13009         /* Groups with a single instruction: no function is generated for this group, the group
13010          * keeps its current instruction. Groups with more than two instructions: one function and
13011          * the associated custom instruction get generated for each such group.
13012          */
13013         TAILQ_FOREACH(g, igl, node) {
13014                 if (g->first_instr_id == g->last_instr_id)
13015                         continue;
13016
13017                 n_custom_instr++;
13018         }
13019
13020         return n_custom_instr;
13021 }
13022
13023 static int
13024 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13025 {
13026         struct action *a;
13027         FILE *f = NULL;
13028
13029         /* Create the .c file. */
13030         f = fopen("/tmp/pipeline.c", "w");
13031         if (!f)
13032                 return -EIO;
13033
13034         /* Include the .h file. */
13035         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
13036
13037         /* Add the code for each action. */
13038         TAILQ_FOREACH(a, &p->actions, node) {
13039                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
13040
13041                 action_data_codegen(a, f);
13042
13043                 fprintf(f, "\n");
13044
13045                 action_instr_codegen(a, f);
13046
13047                 fprintf(f, "\n");
13048         }
13049
13050         /* Add the pipeline code. */
13051         instruction_group_list_codegen(igl, p, f);
13052
13053         /* Close the .c file. */
13054         fclose(f);
13055
13056         return 0;
13057 }
13058
13059 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
13060 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
13061 #endif
13062
13063 static int
13064 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13065 {
13066         struct action *a;
13067         struct instruction_group *g;
13068         char *dir_in, *buffer = NULL;
13069         const char *dir_out;
13070         int status = 0;
13071
13072         /* Get the environment variables. */
13073         dir_in = getenv("RTE_INSTALL_DIR");
13074         if (!dir_in) {
13075                 status = -EINVAL;
13076                 goto free;
13077         }
13078
13079         dir_out = "/tmp";
13080
13081         /* Memory allocation for the command buffer. */
13082         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
13083         if (!buffer) {
13084                 status = -ENOMEM;
13085                 goto free;
13086         }
13087
13088         snprintf(buffer,
13089                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
13090                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
13091                  "-I %s/lib/pipeline "
13092                  "-I %s/lib/eal/include "
13093                  "-I %s/lib/eal/x86/include "
13094                  "-I %s/lib/eal/include/generic "
13095                  "-I %s/lib/meter "
13096                  "-I %s/lib/port "
13097                  "-I %s/lib/table "
13098                  "-I %s/lib/pipeline "
13099                  "-I %s/config "
13100                  "-I %s/build "
13101                  "-I %s/lib/eal/linux/include "
13102                  ">%s/pipeline.log 2>&1 "
13103                  "&& "
13104                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
13105                  ">>%s/pipeline.log 2>&1",
13106                  dir_out,
13107                  dir_out,
13108                  dir_in,
13109                  dir_in,
13110                  dir_in,
13111                  dir_in,
13112                  dir_in,
13113                  dir_in,
13114                  dir_in,
13115                  dir_in,
13116                  dir_in,
13117                  dir_in,
13118                  dir_in,
13119                  dir_out,
13120                  dir_out,
13121                  dir_out,
13122                  dir_out);
13123
13124         /* Build the shared object library. */
13125         status = system(buffer);
13126         if (status)
13127                 goto free;
13128
13129         /* Open library. */
13130         snprintf(buffer,
13131                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
13132                  "%s/libpipeline.so",
13133                  dir_out);
13134
13135         p->lib = dlopen(buffer, RTLD_LAZY);
13136         if (!p->lib) {
13137                 status = -EIO;
13138                 goto free;
13139         }
13140
13141         /* Get the action function symbols. */
13142         TAILQ_FOREACH(a, &p->actions, node) {
13143                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
13144
13145                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
13146                 if (!p->action_funcs[a->id]) {
13147                         status = -EINVAL;
13148                         goto free;
13149                 }
13150         }
13151
13152         /* Get the pipeline function symbols. */
13153         TAILQ_FOREACH(g, igl, node) {
13154                 if (g->first_instr_id == g->last_instr_id)
13155                         continue;
13156
13157                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
13158
13159                 g->func = dlsym(p->lib, buffer);
13160                 if (!g->func) {
13161                         status = -EINVAL;
13162                         goto free;
13163                 }
13164         }
13165
13166 free:
13167         if (status && p->lib) {
13168                 dlclose(p->lib);
13169                 p->lib = NULL;
13170         }
13171
13172         free(buffer);
13173
13174         return status;
13175 }
13176
13177 static int
13178 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
13179                       struct instruction_group_list *igl)
13180 {
13181         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
13182
13183         /* Check that enough space is available within the pipeline instruction table to store all
13184          * the custom instructions.
13185          */
13186         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
13187                 return -ENOSPC;
13188
13189         return 0;
13190 }
13191
13192 static void
13193 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13194 {
13195         struct instruction_group *g;
13196         uint32_t i;
13197
13198         /* Pipeline table instructions. */
13199         for (i = 0; i < p->n_instructions; i++) {
13200                 struct instruction *instr = &p->instructions[i];
13201
13202                 if (instr->type == INSTR_TABLE)
13203                         instr->type = INSTR_TABLE_AF;
13204
13205                 if (instr->type == INSTR_LEARNER)
13206                         instr->type = INSTR_LEARNER_AF;
13207         }
13208
13209         /* Pipeline custom instructions. */
13210         i = 0;
13211         TAILQ_FOREACH(g, igl, node) {
13212                 struct instruction *instr = &p->instructions[g->first_instr_id];
13213                 uint32_t j;
13214
13215                 if (g->first_instr_id == g->last_instr_id)
13216                         continue;
13217
13218                 /* Install a new custom instruction. */
13219                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
13220
13221                 /* First instruction of the group: change its type to the new custom instruction. */
13222                 instr->type = INSTR_CUSTOM_0 + i;
13223
13224                 /* All the subsequent instructions of the group: invalidate. */
13225                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
13226                         struct instruction_data *data = &p->instruction_data[j];
13227
13228                         data->invalid = 1;
13229                 }
13230
13231                 i++;
13232         }
13233
13234         /* Remove the invalidated instructions. */
13235         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
13236
13237         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
13238          * instructions that are the only instruction within their group, so they were left
13239          * unmodified).
13240          */
13241         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
13242 }
13243
13244 static int
13245 pipeline_compile(struct rte_swx_pipeline *p)
13246 {
13247         struct instruction_group_list *igl = NULL;
13248         int status = 0;
13249
13250         igl = instruction_group_list_create(p);
13251         if (!igl) {
13252                 status = -ENOMEM;
13253                 goto free;
13254         }
13255
13256         /* Code generation. */
13257         status = pipeline_codegen(p, igl);
13258         if (status)
13259                 goto free;
13260
13261         /* Build and load the shared object library. */
13262         status = pipeline_libload(p, igl);
13263         if (status)
13264                 goto free;
13265
13266         /* Adjust instructions. */
13267         status = pipeline_adjust_check(p, igl);
13268         if (status)
13269                 goto free;
13270
13271         pipeline_adjust(p, igl);
13272
13273 free:
13274         instruction_group_list_free(igl);
13275
13276         return status;
13277 }