lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <dlfcn.h>
   8
   9 #include <rte_swx_port_ethdev.h>
  10 #include <rte_swx_port_fd.h>
  11 #include <rte_swx_port_ring.h>
  12 #include "rte_swx_port_source_sink.h"
  13
  14 #include <rte_swx_table_em.h>
  15 #include <rte_swx_table_wm.h>
  16
  17 #include "rte_swx_pipeline_internal.h"
  18
  19 #define CHECK(condition, err_code)                                             \
  20 do {                                                                           \
  21         if (!(condition))                                                      \
  22                 return -(err_code);                                            \
  23 } while (0)
  24
  25 #define CHECK_NAME(name, err_code)                                             \
  26         CHECK((name) &&                                                        \
  27               (name)[0] &&                                                     \
  28               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  29               err_code)
  30
  31 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  32         CHECK((instr) &&                                                       \
  33               (instr)[0] &&                                                    \
  34               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  35                RTE_SWX_INSTRUCTION_SIZE),                                      \
  36               err_code)
  37
  38 /*
  39  * Environment.
  40  */
  41 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  42
  43 #include <rte_malloc.h>
  44
  45 static void *
  46 env_malloc(size_t size, size_t alignment, int numa_node)
  47 {
  48         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  49 }
  50
  51 static void
  52 env_free(void *start, size_t size __rte_unused)
  53 {
  54         rte_free(start);
  55 }
  56
  57 #else
  58
  59 #include <numa.h>
  60
  61 static void *
  62 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  63 {
  64         void *start;
  65
  66         if (numa_available() == -1)
  67                 return NULL;
  68
  69         start = numa_alloc_onnode(size, numa_node);
  70         if (!start)
  71                 return NULL;
  72
  73         memset(start, 0, size);
  74         return start;
  75 }
  76
  77 static void
  78 env_free(void *start, size_t size)
  79 {
  80         if (numa_available() == -1)
  81                 return;
  82
  83         numa_free(start, size);
  84 }
  85
  86 #endif
  87
  88 /*
  89  * Struct.
  90  */
  91 static struct struct_type *
  92 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  93 {
  94         struct struct_type *elem;
  95
  96         TAILQ_FOREACH(elem, &p->struct_types, node)
  97                 if (strcmp(elem->name, name) == 0)
  98                         return elem;
  99
 100         return NULL;
 101 }
 102
 103 static struct field *
 104 struct_type_field_find(struct struct_type *st, const char *name)
 105 {
 106         uint32_t i;
 107
 108         for (i = 0; i < st->n_fields; i++) {
 109                 struct field *f = &st->fields[i];
 110
 111                 if (strcmp(f->name, name) == 0)
 112                         return f;
 113         }
 114
 115         return NULL;
 116 }
 117
 118 int
 119 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 120                                       const char *name,
 121                                       struct rte_swx_field_params *fields,
 122                                       uint32_t n_fields,
 123                                       int last_field_has_variable_size)
 124 {
 125         struct struct_type *st;
 126         uint32_t i;
 127
 128         CHECK(p, EINVAL);
 129         CHECK_NAME(name, EINVAL);
 130         CHECK(fields, EINVAL);
 131         CHECK(n_fields, EINVAL);
 132
 133         for (i = 0; i < n_fields; i++) {
 134                 struct rte_swx_field_params *f = &fields[i];
 135                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 136                 uint32_t j;
 137
 138                 CHECK_NAME(f->name, EINVAL);
 139                 CHECK(f->n_bits, EINVAL);
 140                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 141                 CHECK((f->n_bits & 7) == 0, EINVAL);
 142
 143                 for (j = 0; j < i; j++) {
 144                         struct rte_swx_field_params *f_prev = &fields[j];
 145
 146                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 147                 }
 148         }
 149
 150         CHECK(!struct_type_find(p, name), EEXIST);
 151
 152         /* Node allocation. */
 153         st = calloc(1, sizeof(struct struct_type));
 154         CHECK(st, ENOMEM);
 155
 156         st->fields = calloc(n_fields, sizeof(struct field));
 157         if (!st->fields) {
 158                 free(st);
 159                 CHECK(0, ENOMEM);
 160         }
 161
 162         /* Node initialization. */
 163         strcpy(st->name, name);
 164         for (i = 0; i < n_fields; i++) {
 165                 struct field *dst = &st->fields[i];
 166                 struct rte_swx_field_params *src = &fields[i];
 167                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 168
 169                 strcpy(dst->name, src->name);
 170                 dst->n_bits = src->n_bits;
 171                 dst->offset = st->n_bits;
 172                 dst->var_size = var_size;
 173
 174                 st->n_bits += src->n_bits;
 175                 st->n_bits_min += var_size ? 0 : src->n_bits;
 176         }
 177         st->n_fields = n_fields;
 178         st->var_size = last_field_has_variable_size;
 179
 180         /* Node add to tailq. */
 181         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 182
 183         return 0;
 184 }
 185
 186 static int
 187 struct_build(struct rte_swx_pipeline *p)
 188 {
 189         uint32_t i;
 190
 191         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 192                 struct thread *t = &p->threads[i];
 193
 194                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 195                 CHECK(t->structs, ENOMEM);
 196         }
 197
 198         return 0;
 199 }
 200
 201 static void
 202 struct_build_free(struct rte_swx_pipeline *p)
 203 {
 204         uint32_t i;
 205
 206         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 207                 struct thread *t = &p->threads[i];
 208
 209                 free(t->structs);
 210                 t->structs = NULL;
 211         }
 212 }
 213
 214 static void
 215 struct_free(struct rte_swx_pipeline *p)
 216 {
 217         struct_build_free(p);
 218
 219         /* Struct types. */
 220         for ( ; ; ) {
 221                 struct struct_type *elem;
 222
 223                 elem = TAILQ_FIRST(&p->struct_types);
 224                 if (!elem)
 225                         break;
 226
 227                 TAILQ_REMOVE(&p->struct_types, elem, node);
 228                 free(elem->fields);
 229                 free(elem);
 230         }
 231 }
 232
 233 /*
 234  * Input port.
 235  */
 236 static struct port_in_type *
 237 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 238 {
 239         struct port_in_type *elem;
 240
 241         if (!name)
 242                 return NULL;
 243
 244         TAILQ_FOREACH(elem, &p->port_in_types, node)
 245                 if (strcmp(elem->name, name) == 0)
 246                         return elem;
 247
 248         return NULL;
 249 }
 250
 251 int
 252 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 253                                        const char *name,
 254                                        struct rte_swx_port_in_ops *ops)
 255 {
 256         struct port_in_type *elem;
 257
 258         CHECK(p, EINVAL);
 259         CHECK_NAME(name, EINVAL);
 260         CHECK(ops, EINVAL);
 261         CHECK(ops->create, EINVAL);
 262         CHECK(ops->free, EINVAL);
 263         CHECK(ops->pkt_rx, EINVAL);
 264         CHECK(ops->stats_read, EINVAL);
 265
 266         CHECK(!port_in_type_find(p, name), EEXIST);
 267
 268         /* Node allocation. */
 269         elem = calloc(1, sizeof(struct port_in_type));
 270         CHECK(elem, ENOMEM);
 271
 272         /* Node initialization. */
 273         strcpy(elem->name, name);
 274         memcpy(&elem->ops, ops, sizeof(*ops));
 275
 276         /* Node add to tailq. */
 277         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 278
 279         return 0;
 280 }
 281
 282 static struct port_in *
 283 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 284 {
 285         struct port_in *port;
 286
 287         TAILQ_FOREACH(port, &p->ports_in, node)
 288                 if (port->id == port_id)
 289                         return port;
 290
 291         return NULL;
 292 }
 293
 294 int
 295 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 296                                 uint32_t port_id,
 297                                 const char *port_type_name,
 298                                 void *args)
 299 {
 300         struct port_in_type *type = NULL;
 301         struct port_in *port = NULL;
 302         void *obj = NULL;
 303
 304         CHECK(p, EINVAL);
 305
 306         CHECK(!port_in_find(p, port_id), EINVAL);
 307
 308         CHECK_NAME(port_type_name, EINVAL);
 309         type = port_in_type_find(p, port_type_name);
 310         CHECK(type, EINVAL);
 311
 312         obj = type->ops.create(args);
 313         CHECK(obj, ENODEV);
 314
 315         /* Node allocation. */
 316         port = calloc(1, sizeof(struct port_in));
 317         CHECK(port, ENOMEM);
 318
 319         /* Node initialization. */
 320         port->type = type;
 321         port->obj = obj;
 322         port->id = port_id;
 323
 324         /* Node add to tailq. */
 325         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 326         if (p->n_ports_in < port_id + 1)
 327                 p->n_ports_in = port_id + 1;
 328
 329         return 0;
 330 }
 331
 332 static int
 333 port_in_build(struct rte_swx_pipeline *p)
 334 {
 335         struct port_in *port;
 336         uint32_t i;
 337
 338         CHECK(p->n_ports_in, EINVAL);
 339         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 340
 341         for (i = 0; i < p->n_ports_in; i++)
 342                 CHECK(port_in_find(p, i), EINVAL);
 343
 344         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 345         CHECK(p->in, ENOMEM);
 346
 347         TAILQ_FOREACH(port, &p->ports_in, node) {
 348                 struct port_in_runtime *in = &p->in[port->id];
 349
 350                 in->pkt_rx = port->type->ops.pkt_rx;
 351                 in->obj = port->obj;
 352         }
 353
 354         return 0;
 355 }
 356
 357 static void
 358 port_in_build_free(struct rte_swx_pipeline *p)
 359 {
 360         free(p->in);
 361         p->in = NULL;
 362 }
 363
 364 static void
 365 port_in_free(struct rte_swx_pipeline *p)
 366 {
 367         port_in_build_free(p);
 368
 369         /* Input ports. */
 370         for ( ; ; ) {
 371                 struct port_in *port;
 372
 373                 port = TAILQ_FIRST(&p->ports_in);
 374                 if (!port)
 375                         break;
 376
 377                 TAILQ_REMOVE(&p->ports_in, port, node);
 378                 port->type->ops.free(port->obj);
 379                 free(port);
 380         }
 381
 382         /* Input port types. */
 383         for ( ; ; ) {
 384                 struct port_in_type *elem;
 385
 386                 elem = TAILQ_FIRST(&p->port_in_types);
 387                 if (!elem)
 388                         break;
 389
 390                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 391                 free(elem);
 392         }
 393 }
 394
 395 /*
 396  * Output port.
 397  */
 398 static struct port_out_type *
 399 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 400 {
 401         struct port_out_type *elem;
 402
 403         if (!name)
 404                 return NULL;
 405
 406         TAILQ_FOREACH(elem, &p->port_out_types, node)
 407                 if (!strcmp(elem->name, name))
 408                         return elem;
 409
 410         return NULL;
 411 }
 412
 413 int
 414 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 415                                         const char *name,
 416                                         struct rte_swx_port_out_ops *ops)
 417 {
 418         struct port_out_type *elem;
 419
 420         CHECK(p, EINVAL);
 421         CHECK_NAME(name, EINVAL);
 422         CHECK(ops, EINVAL);
 423         CHECK(ops->create, EINVAL);
 424         CHECK(ops->free, EINVAL);
 425         CHECK(ops->pkt_tx, EINVAL);
 426         CHECK(ops->pkt_fast_clone_tx, EINVAL);
 427         CHECK(ops->pkt_clone_tx, EINVAL);
 428         CHECK(ops->stats_read, EINVAL);
 429
 430         CHECK(!port_out_type_find(p, name), EEXIST);
 431
 432         /* Node allocation. */
 433         elem = calloc(1, sizeof(struct port_out_type));
 434         CHECK(elem, ENOMEM);
 435
 436         /* Node initialization. */
 437         strcpy(elem->name, name);
 438         memcpy(&elem->ops, ops, sizeof(*ops));
 439
 440         /* Node add to tailq. */
 441         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 442
 443         return 0;
 444 }
 445
 446 static struct port_out *
 447 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 448 {
 449         struct port_out *port;
 450
 451         TAILQ_FOREACH(port, &p->ports_out, node)
 452                 if (port->id == port_id)
 453                         return port;
 454
 455         return NULL;
 456 }
 457
 458 int
 459 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 460                                  uint32_t port_id,
 461                                  const char *port_type_name,
 462                                  void *args)
 463 {
 464         struct port_out_type *type = NULL;
 465         struct port_out *port = NULL;
 466         void *obj = NULL;
 467
 468         CHECK(p, EINVAL);
 469
 470         CHECK(!port_out_find(p, port_id), EINVAL);
 471
 472         CHECK_NAME(port_type_name, EINVAL);
 473         type = port_out_type_find(p, port_type_name);
 474         CHECK(type, EINVAL);
 475
 476         obj = type->ops.create(args);
 477         CHECK(obj, ENODEV);
 478
 479         /* Node allocation. */
 480         port = calloc(1, sizeof(struct port_out));
 481         CHECK(port, ENOMEM);
 482
 483         /* Node initialization. */
 484         port->type = type;
 485         port->obj = obj;
 486         port->id = port_id;
 487
 488         /* Node add to tailq. */
 489         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 490         if (p->n_ports_out < port_id + 1)
 491                 p->n_ports_out = port_id + 1;
 492
 493         return 0;
 494 }
 495
 496 static int
 497 port_out_build(struct rte_swx_pipeline *p)
 498 {
 499         struct port_out *port;
 500         uint32_t i;
 501
 502         CHECK(p->n_ports_out, EINVAL);
 503
 504         for (i = 0; i < p->n_ports_out; i++)
 505                 CHECK(port_out_find(p, i), EINVAL);
 506
 507         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 508         CHECK(p->out, ENOMEM);
 509
 510         TAILQ_FOREACH(port, &p->ports_out, node) {
 511                 struct port_out_runtime *out = &p->out[port->id];
 512
 513                 out->pkt_tx = port->type->ops.pkt_tx;
 514                 out->pkt_fast_clone_tx = port->type->ops.pkt_fast_clone_tx;
 515                 out->pkt_clone_tx = port->type->ops.pkt_clone_tx;
 516                 out->flush = port->type->ops.flush;
 517                 out->obj = port->obj;
 518         }
 519
 520         return 0;
 521 }
 522
 523 static void
 524 port_out_build_free(struct rte_swx_pipeline *p)
 525 {
 526         free(p->out);
 527         p->out = NULL;
 528 }
 529
 530 static void
 531 port_out_free(struct rte_swx_pipeline *p)
 532 {
 533         port_out_build_free(p);
 534
 535         /* Output ports. */
 536         for ( ; ; ) {
 537                 struct port_out *port;
 538
 539                 port = TAILQ_FIRST(&p->ports_out);
 540                 if (!port)
 541                         break;
 542
 543                 TAILQ_REMOVE(&p->ports_out, port, node);
 544                 port->type->ops.free(port->obj);
 545                 free(port);
 546         }
 547
 548         /* Output port types. */
 549         for ( ; ; ) {
 550                 struct port_out_type *elem;
 551
 552                 elem = TAILQ_FIRST(&p->port_out_types);
 553                 if (!elem)
 554                         break;
 555
 556                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 557                 free(elem);
 558         }
 559 }
 560
 561 /*
 562  * Packet mirroring.
 563  */
 564 int
 565 rte_swx_pipeline_mirroring_config(struct rte_swx_pipeline *p,
 566                                   struct rte_swx_pipeline_mirroring_params *params)
 567 {
 568         CHECK(p, EINVAL);
 569         CHECK(params, EINVAL);
 570         CHECK(params->n_slots, EINVAL);
 571         CHECK(params->n_sessions, EINVAL);
 572         CHECK(!p->build_done, EEXIST);
 573
 574         p->n_mirroring_slots = rte_align32pow2(params->n_slots);
 575         if (p->n_mirroring_slots > 64)
 576                 p->n_mirroring_slots = 64;
 577
 578         p->n_mirroring_sessions = rte_align32pow2(params->n_sessions);
 579
 580         return 0;
 581 }
 582
 583 static void
 584 mirroring_build_free(struct rte_swx_pipeline *p)
 585 {
 586         uint32_t i;
 587
 588         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 589                 struct thread *t = &p->threads[i];
 590
 591                 /* mirroring_slots. */
 592                 free(t->mirroring_slots);
 593                 t->mirroring_slots = NULL;
 594         }
 595
 596         /* mirroring_sessions. */
 597         free(p->mirroring_sessions);
 598         p->mirroring_sessions = NULL;
 599 }
 600
 601 static void
 602 mirroring_free(struct rte_swx_pipeline *p)
 603 {
 604         mirroring_build_free(p);
 605 }
 606
 607 static int
 608 mirroring_build(struct rte_swx_pipeline *p)
 609 {
 610         uint32_t i;
 611
 612         if (!p->n_mirroring_slots || !p->n_mirroring_sessions)
 613                 return 0;
 614
 615         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 616                 struct thread *t = &p->threads[i];
 617
 618                 /* mirroring_slots. */
 619                 t->mirroring_slots = calloc(p->n_mirroring_slots, sizeof(uint32_t));
 620                 if (!t->mirroring_slots)
 621                         goto error;
 622         }
 623
 624         /* mirroring_sessions. */
 625         p->mirroring_sessions = calloc(p->n_mirroring_sessions, sizeof(struct mirroring_session));
 626         if (!p->mirroring_sessions)
 627                 goto error;
 628
 629         return 0;
 630
 631 error:
 632         mirroring_build_free(p);
 633         return -ENOMEM;
 634 }
 635
 636 /*
 637  * Extern object.
 638  */
 639 static struct extern_type *
 640 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 641 {
 642         struct extern_type *elem;
 643
 644         TAILQ_FOREACH(elem, &p->extern_types, node)
 645                 if (strcmp(elem->name, name) == 0)
 646                         return elem;
 647
 648         return NULL;
 649 }
 650
 651 static struct extern_type_member_func *
 652 extern_type_member_func_find(struct extern_type *type, const char *name)
 653 {
 654         struct extern_type_member_func *elem;
 655
 656         TAILQ_FOREACH(elem, &type->funcs, node)
 657                 if (strcmp(elem->name, name) == 0)
 658                         return elem;
 659
 660         return NULL;
 661 }
 662
 663 static struct extern_obj *
 664 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 665 {
 666         struct extern_obj *elem;
 667
 668         TAILQ_FOREACH(elem, &p->extern_objs, node)
 669                 if (strcmp(elem->name, name) == 0)
 670                         return elem;
 671
 672         return NULL;
 673 }
 674
 675 static struct extern_type_member_func *
 676 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 677                              const char *name,
 678                              struct extern_obj **obj)
 679 {
 680         struct extern_obj *object;
 681         struct extern_type_member_func *func;
 682         char *object_name, *func_name;
 683
 684         if (name[0] != 'e' || name[1] != '.')
 685                 return NULL;
 686
 687         object_name = strdup(&name[2]);
 688         if (!object_name)
 689                 return NULL;
 690
 691         func_name = strchr(object_name, '.');
 692         if (!func_name) {
 693                 free(object_name);
 694                 return NULL;
 695         }
 696
 697         *func_name = 0;
 698         func_name++;
 699
 700         object = extern_obj_find(p, object_name);
 701         if (!object) {
 702                 free(object_name);
 703                 return NULL;
 704         }
 705
 706         func = extern_type_member_func_find(object->type, func_name);
 707         if (!func) {
 708                 free(object_name);
 709                 return NULL;
 710         }
 711
 712         if (obj)
 713                 *obj = object;
 714
 715         free(object_name);
 716         return func;
 717 }
 718
 719 static struct field *
 720 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 721                                const char *name,
 722                                struct extern_obj **object)
 723 {
 724         struct extern_obj *obj;
 725         struct field *f;
 726         char *obj_name, *field_name;
 727
 728         if ((name[0] != 'e') || (name[1] != '.'))
 729                 return NULL;
 730
 731         obj_name = strdup(&name[2]);
 732         if (!obj_name)
 733                 return NULL;
 734
 735         field_name = strchr(obj_name, '.');
 736         if (!field_name) {
 737                 free(obj_name);
 738                 return NULL;
 739         }
 740
 741         *field_name = 0;
 742         field_name++;
 743
 744         obj = extern_obj_find(p, obj_name);
 745         if (!obj) {
 746                 free(obj_name);
 747                 return NULL;
 748         }
 749
 750         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 751         if (!f) {
 752                 free(obj_name);
 753                 return NULL;
 754         }
 755
 756         if (object)
 757                 *object = obj;
 758
 759         free(obj_name);
 760         return f;
 761 }
 762
 763 int
 764 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 765         const char *name,
 766         const char *mailbox_struct_type_name,
 767         rte_swx_extern_type_constructor_t constructor,
 768         rte_swx_extern_type_destructor_t destructor)
 769 {
 770         struct extern_type *elem;
 771         struct struct_type *mailbox_struct_type;
 772
 773         CHECK(p, EINVAL);
 774
 775         CHECK_NAME(name, EINVAL);
 776         CHECK(!extern_type_find(p, name), EEXIST);
 777
 778         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 779         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 780         CHECK(mailbox_struct_type, EINVAL);
 781         CHECK(!mailbox_struct_type->var_size, EINVAL);
 782
 783         CHECK(constructor, EINVAL);
 784         CHECK(destructor, EINVAL);
 785
 786         /* Node allocation. */
 787         elem = calloc(1, sizeof(struct extern_type));
 788         CHECK(elem, ENOMEM);
 789
 790         /* Node initialization. */
 791         strcpy(elem->name, name);
 792         elem->mailbox_struct_type = mailbox_struct_type;
 793         elem->constructor = constructor;
 794         elem->destructor = destructor;
 795         TAILQ_INIT(&elem->funcs);
 796
 797         /* Node add to tailq. */
 798         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 799
 800         return 0;
 801 }
 802
 803 int
 804 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 805         const char *extern_type_name,
 806         const char *name,
 807         rte_swx_extern_type_member_func_t member_func)
 808 {
 809         struct extern_type *type;
 810         struct extern_type_member_func *type_member;
 811
 812         CHECK(p, EINVAL);
 813
 814         CHECK_NAME(extern_type_name, EINVAL);
 815         type = extern_type_find(p, extern_type_name);
 816         CHECK(type, EINVAL);
 817         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 818
 819         CHECK_NAME(name, EINVAL);
 820         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 821
 822         CHECK(member_func, EINVAL);
 823
 824         /* Node allocation. */
 825         type_member = calloc(1, sizeof(struct extern_type_member_func));
 826         CHECK(type_member, ENOMEM);
 827
 828         /* Node initialization. */
 829         strcpy(type_member->name, name);
 830         type_member->func = member_func;
 831         type_member->id = type->n_funcs;
 832
 833         /* Node add to tailq. */
 834         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 835         type->n_funcs++;
 836
 837         return 0;
 838 }
 839
 840 int
 841 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 842                                       const char *extern_type_name,
 843                                       const char *name,
 844                                       const char *args)
 845 {
 846         struct extern_type *type;
 847         struct extern_obj *obj;
 848         void *obj_handle;
 849
 850         CHECK(p, EINVAL);
 851
 852         CHECK_NAME(extern_type_name, EINVAL);
 853         type = extern_type_find(p, extern_type_name);
 854         CHECK(type, EINVAL);
 855
 856         CHECK_NAME(name, EINVAL);
 857         CHECK(!extern_obj_find(p, name), EEXIST);
 858
 859         /* Node allocation. */
 860         obj = calloc(1, sizeof(struct extern_obj));
 861         CHECK(obj, ENOMEM);
 862
 863         /* Object construction. */
 864         obj_handle = type->constructor(args);
 865         if (!obj_handle) {
 866                 free(obj);
 867                 CHECK(0, ENODEV);
 868         }
 869
 870         /* Node initialization. */
 871         strcpy(obj->name, name);
 872         obj->type = type;
 873         obj->obj = obj_handle;
 874         obj->struct_id = p->n_structs;
 875         obj->id = p->n_extern_objs;
 876
 877         /* Node add to tailq. */
 878         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 879         p->n_extern_objs++;
 880         p->n_structs++;
 881
 882         return 0;
 883 }
 884
 885 static int
 886 extern_obj_build(struct rte_swx_pipeline *p)
 887 {
 888         uint32_t i;
 889
 890         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 891                 struct thread *t = &p->threads[i];
 892                 struct extern_obj *obj;
 893
 894                 t->extern_objs = calloc(p->n_extern_objs,
 895                                         sizeof(struct extern_obj_runtime));
 896                 CHECK(t->extern_objs, ENOMEM);
 897
 898                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 899                         struct extern_obj_runtime *r =
 900                                 &t->extern_objs[obj->id];
 901                         struct extern_type_member_func *func;
 902                         uint32_t mailbox_size =
 903                                 obj->type->mailbox_struct_type->n_bits / 8;
 904
 905                         r->obj = obj->obj;
 906
 907                         r->mailbox = calloc(1, mailbox_size);
 908                         CHECK(r->mailbox, ENOMEM);
 909
 910                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 911                                 r->funcs[func->id] = func->func;
 912
 913                         t->structs[obj->struct_id] = r->mailbox;
 914                 }
 915         }
 916
 917         return 0;
 918 }
 919
 920 static void
 921 extern_obj_build_free(struct rte_swx_pipeline *p)
 922 {
 923         uint32_t i;
 924
 925         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 926                 struct thread *t = &p->threads[i];
 927                 uint32_t j;
 928
 929                 if (!t->extern_objs)
 930                         continue;
 931
 932                 for (j = 0; j < p->n_extern_objs; j++) {
 933                         struct extern_obj_runtime *r = &t->extern_objs[j];
 934
 935                         free(r->mailbox);
 936                 }
 937
 938                 free(t->extern_objs);
 939                 t->extern_objs = NULL;
 940         }
 941 }
 942
 943 static void
 944 extern_obj_free(struct rte_swx_pipeline *p)
 945 {
 946         extern_obj_build_free(p);
 947
 948         /* Extern objects. */
 949         for ( ; ; ) {
 950                 struct extern_obj *elem;
 951
 952                 elem = TAILQ_FIRST(&p->extern_objs);
 953                 if (!elem)
 954                         break;
 955
 956                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 957                 if (elem->obj)
 958                         elem->type->destructor(elem->obj);
 959                 free(elem);
 960         }
 961
 962         /* Extern types. */
 963         for ( ; ; ) {
 964                 struct extern_type *elem;
 965
 966                 elem = TAILQ_FIRST(&p->extern_types);
 967                 if (!elem)
 968                         break;
 969
 970                 TAILQ_REMOVE(&p->extern_types, elem, node);
 971
 972                 for ( ; ; ) {
 973                         struct extern_type_member_func *func;
 974
 975                         func = TAILQ_FIRST(&elem->funcs);
 976                         if (!func)
 977                                 break;
 978
 979                         TAILQ_REMOVE(&elem->funcs, func, node);
 980                         free(func);
 981                 }
 982
 983                 free(elem);
 984         }
 985 }
 986
 987 /*
 988  * Extern function.
 989  */
 990 static struct extern_func *
 991 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 992 {
 993         struct extern_func *elem;
 994
 995         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 996                 if (strcmp(elem->name, name) == 0)
 997                         return elem;
 998
 999         return NULL;
1000 }
1001
1002 static struct extern_func *
1003 extern_func_parse(struct rte_swx_pipeline *p,
1004                   const char *name)
1005 {
1006         if (name[0] != 'f' || name[1] != '.')
1007                 return NULL;
1008
1009         return extern_func_find(p, &name[2]);
1010 }
1011
1012 static struct field *
1013 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1014                                 const char *name,
1015                                 struct extern_func **function)
1016 {
1017         struct extern_func *func;
1018         struct field *f;
1019         char *func_name, *field_name;
1020
1021         if ((name[0] != 'f') || (name[1] != '.'))
1022                 return NULL;
1023
1024         func_name = strdup(&name[2]);
1025         if (!func_name)
1026                 return NULL;
1027
1028         field_name = strchr(func_name, '.');
1029         if (!field_name) {
1030                 free(func_name);
1031                 return NULL;
1032         }
1033
1034         *field_name = 0;
1035         field_name++;
1036
1037         func = extern_func_find(p, func_name);
1038         if (!func) {
1039                 free(func_name);
1040                 return NULL;
1041         }
1042
1043         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1044         if (!f) {
1045                 free(func_name);
1046                 return NULL;
1047         }
1048
1049         if (function)
1050                 *function = func;
1051
1052         free(func_name);
1053         return f;
1054 }
1055
1056 int
1057 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1058                                       const char *name,
1059                                       const char *mailbox_struct_type_name,
1060                                       rte_swx_extern_func_t func)
1061 {
1062         struct extern_func *f;
1063         struct struct_type *mailbox_struct_type;
1064
1065         CHECK(p, EINVAL);
1066
1067         CHECK_NAME(name, EINVAL);
1068         CHECK(!extern_func_find(p, name), EEXIST);
1069
1070         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1071         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1072         CHECK(mailbox_struct_type, EINVAL);
1073         CHECK(!mailbox_struct_type->var_size, EINVAL);
1074
1075         CHECK(func, EINVAL);
1076
1077         /* Node allocation. */
1078         f = calloc(1, sizeof(struct extern_func));
1079         CHECK(func, ENOMEM);
1080
1081         /* Node initialization. */
1082         strcpy(f->name, name);
1083         f->mailbox_struct_type = mailbox_struct_type;
1084         f->func = func;
1085         f->struct_id = p->n_structs;
1086         f->id = p->n_extern_funcs;
1087
1088         /* Node add to tailq. */
1089         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1090         p->n_extern_funcs++;
1091         p->n_structs++;
1092
1093         return 0;
1094 }
1095
1096 static int
1097 extern_func_build(struct rte_swx_pipeline *p)
1098 {
1099         uint32_t i;
1100
1101         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1102                 struct thread *t = &p->threads[i];
1103                 struct extern_func *func;
1104
1105                 /* Memory allocation. */
1106                 t->extern_funcs = calloc(p->n_extern_funcs,
1107                                          sizeof(struct extern_func_runtime));
1108                 CHECK(t->extern_funcs, ENOMEM);
1109
1110                 /* Extern function. */
1111                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1112                         struct extern_func_runtime *r =
1113                                 &t->extern_funcs[func->id];
1114                         uint32_t mailbox_size =
1115                                 func->mailbox_struct_type->n_bits / 8;
1116
1117                         r->func = func->func;
1118
1119                         r->mailbox = calloc(1, mailbox_size);
1120                         CHECK(r->mailbox, ENOMEM);
1121
1122                         t->structs[func->struct_id] = r->mailbox;
1123                 }
1124         }
1125
1126         return 0;
1127 }
1128
1129 static void
1130 extern_func_build_free(struct rte_swx_pipeline *p)
1131 {
1132         uint32_t i;
1133
1134         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1135                 struct thread *t = &p->threads[i];
1136                 uint32_t j;
1137
1138                 if (!t->extern_funcs)
1139                         continue;
1140
1141                 for (j = 0; j < p->n_extern_funcs; j++) {
1142                         struct extern_func_runtime *r = &t->extern_funcs[j];
1143
1144                         free(r->mailbox);
1145                 }
1146
1147                 free(t->extern_funcs);
1148                 t->extern_funcs = NULL;
1149         }
1150 }
1151
1152 static void
1153 extern_func_free(struct rte_swx_pipeline *p)
1154 {
1155         extern_func_build_free(p);
1156
1157         for ( ; ; ) {
1158                 struct extern_func *elem;
1159
1160                 elem = TAILQ_FIRST(&p->extern_funcs);
1161                 if (!elem)
1162                         break;
1163
1164                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1165                 free(elem);
1166         }
1167 }
1168
1169 /*
1170  * Header.
1171  */
1172 static struct header *
1173 header_find(struct rte_swx_pipeline *p, const char *name)
1174 {
1175         struct header *elem;
1176
1177         TAILQ_FOREACH(elem, &p->headers, node)
1178                 if (strcmp(elem->name, name) == 0)
1179                         return elem;
1180
1181         return NULL;
1182 }
1183
1184 static struct header *
1185 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1186 {
1187         struct header *elem;
1188
1189         TAILQ_FOREACH(elem, &p->headers, node)
1190                 if (elem->struct_id == struct_id)
1191                         return elem;
1192
1193         return NULL;
1194 }
1195
1196 static struct header *
1197 header_parse(struct rte_swx_pipeline *p,
1198              const char *name)
1199 {
1200         if (name[0] != 'h' || name[1] != '.')
1201                 return NULL;
1202
1203         return header_find(p, &name[2]);
1204 }
1205
1206 static struct field *
1207 header_field_parse(struct rte_swx_pipeline *p,
1208                    const char *name,
1209                    struct header **header)
1210 {
1211         struct header *h;
1212         struct field *f;
1213         char *header_name, *field_name;
1214
1215         if ((name[0] != 'h') || (name[1] != '.'))
1216                 return NULL;
1217
1218         header_name = strdup(&name[2]);
1219         if (!header_name)
1220                 return NULL;
1221
1222         field_name = strchr(header_name, '.');
1223         if (!field_name) {
1224                 free(header_name);
1225                 return NULL;
1226         }
1227
1228         *field_name = 0;
1229         field_name++;
1230
1231         h = header_find(p, header_name);
1232         if (!h) {
1233                 free(header_name);
1234                 return NULL;
1235         }
1236
1237         f = struct_type_field_find(h->st, field_name);
1238         if (!f) {
1239                 free(header_name);
1240                 return NULL;
1241         }
1242
1243         if (header)
1244                 *header = h;
1245
1246         free(header_name);
1247         return f;
1248 }
1249
1250 int
1251 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1252                                         const char *name,
1253                                         const char *struct_type_name)
1254 {
1255         struct struct_type *st;
1256         struct header *h;
1257         size_t n_headers_max;
1258
1259         CHECK(p, EINVAL);
1260         CHECK_NAME(name, EINVAL);
1261         CHECK_NAME(struct_type_name, EINVAL);
1262
1263         CHECK(!header_find(p, name), EEXIST);
1264
1265         st = struct_type_find(p, struct_type_name);
1266         CHECK(st, EINVAL);
1267
1268         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1269         CHECK(p->n_headers < n_headers_max, ENOSPC);
1270
1271         /* Node allocation. */
1272         h = calloc(1, sizeof(struct header));
1273         CHECK(h, ENOMEM);
1274
1275         /* Node initialization. */
1276         strcpy(h->name, name);
1277         h->st = st;
1278         h->struct_id = p->n_structs;
1279         h->id = p->n_headers;
1280
1281         /* Node add to tailq. */
1282         TAILQ_INSERT_TAIL(&p->headers, h, node);
1283         p->n_headers++;
1284         p->n_structs++;
1285
1286         return 0;
1287 }
1288
1289 static int
1290 header_build(struct rte_swx_pipeline *p)
1291 {
1292         struct header *h;
1293         uint32_t n_bytes = 0, i;
1294
1295         TAILQ_FOREACH(h, &p->headers, node) {
1296                 n_bytes += h->st->n_bits / 8;
1297         }
1298
1299         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1300                 struct thread *t = &p->threads[i];
1301                 uint32_t offset = 0;
1302
1303                 t->headers = calloc(p->n_headers,
1304                                     sizeof(struct header_runtime));
1305                 CHECK(t->headers, ENOMEM);
1306
1307                 t->headers_out = calloc(p->n_headers,
1308                                         sizeof(struct header_out_runtime));
1309                 CHECK(t->headers_out, ENOMEM);
1310
1311                 t->header_storage = calloc(1, n_bytes);
1312                 CHECK(t->header_storage, ENOMEM);
1313
1314                 t->header_out_storage = calloc(1, n_bytes);
1315                 CHECK(t->header_out_storage, ENOMEM);
1316
1317                 TAILQ_FOREACH(h, &p->headers, node) {
1318                         uint8_t *header_storage;
1319                         uint32_t n_bytes =  h->st->n_bits / 8;
1320
1321                         header_storage = &t->header_storage[offset];
1322                         offset += n_bytes;
1323
1324                         t->headers[h->id].ptr0 = header_storage;
1325                         t->headers[h->id].n_bytes = n_bytes;
1326
1327                         t->structs[h->struct_id] = header_storage;
1328                 }
1329         }
1330
1331         return 0;
1332 }
1333
1334 static void
1335 header_build_free(struct rte_swx_pipeline *p)
1336 {
1337         uint32_t i;
1338
1339         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1340                 struct thread *t = &p->threads[i];
1341
1342                 free(t->headers_out);
1343                 t->headers_out = NULL;
1344
1345                 free(t->headers);
1346                 t->headers = NULL;
1347
1348                 free(t->header_out_storage);
1349                 t->header_out_storage = NULL;
1350
1351                 free(t->header_storage);
1352                 t->header_storage = NULL;
1353         }
1354 }
1355
1356 static void
1357 header_free(struct rte_swx_pipeline *p)
1358 {
1359         header_build_free(p);
1360
1361         for ( ; ; ) {
1362                 struct header *elem;
1363
1364                 elem = TAILQ_FIRST(&p->headers);
1365                 if (!elem)
1366                         break;
1367
1368                 TAILQ_REMOVE(&p->headers, elem, node);
1369                 free(elem);
1370         }
1371 }
1372
1373 /*
1374  * Meta-data.
1375  */
1376 static struct field *
1377 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1378 {
1379         if (!p->metadata_st)
1380                 return NULL;
1381
1382         if (name[0] != 'm' || name[1] != '.')
1383                 return NULL;
1384
1385         return struct_type_field_find(p->metadata_st, &name[2]);
1386 }
1387
1388 int
1389 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1390                                           const char *struct_type_name)
1391 {
1392         struct struct_type *st = NULL;
1393
1394         CHECK(p, EINVAL);
1395
1396         CHECK_NAME(struct_type_name, EINVAL);
1397         st  = struct_type_find(p, struct_type_name);
1398         CHECK(st, EINVAL);
1399         CHECK(!st->var_size, EINVAL);
1400         CHECK(!p->metadata_st, EINVAL);
1401
1402         p->metadata_st = st;
1403         p->metadata_struct_id = p->n_structs;
1404
1405         p->n_structs++;
1406
1407         return 0;
1408 }
1409
1410 static int
1411 metadata_build(struct rte_swx_pipeline *p)
1412 {
1413         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1414         uint32_t i;
1415
1416         /* Thread-level initialization. */
1417         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1418                 struct thread *t = &p->threads[i];
1419                 uint8_t *metadata;
1420
1421                 metadata = calloc(1, n_bytes);
1422                 CHECK(metadata, ENOMEM);
1423
1424                 t->metadata = metadata;
1425                 t->structs[p->metadata_struct_id] = metadata;
1426         }
1427
1428         return 0;
1429 }
1430
1431 static void
1432 metadata_build_free(struct rte_swx_pipeline *p)
1433 {
1434         uint32_t i;
1435
1436         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1437                 struct thread *t = &p->threads[i];
1438
1439                 free(t->metadata);
1440                 t->metadata = NULL;
1441         }
1442 }
1443
1444 static void
1445 metadata_free(struct rte_swx_pipeline *p)
1446 {
1447         metadata_build_free(p);
1448 }
1449
1450 /*
1451  * Instruction.
1452  */
1453 static int
1454 instruction_is_tx(enum instruction_type type)
1455 {
1456         switch (type) {
1457         case INSTR_TX:
1458         case INSTR_TX_I:
1459         case INSTR_DROP:
1460                 return 1;
1461
1462         default:
1463                 return 0;
1464         }
1465 }
1466
1467 static int
1468 instruction_does_tx(struct instruction *instr)
1469 {
1470         switch (instr->type) {
1471         case INSTR_TX:
1472         case INSTR_TX_I:
1473         case INSTR_DROP:
1474         case INSTR_HDR_EMIT_TX:
1475         case INSTR_HDR_EMIT2_TX:
1476         case INSTR_HDR_EMIT3_TX:
1477         case INSTR_HDR_EMIT4_TX:
1478         case INSTR_HDR_EMIT5_TX:
1479         case INSTR_HDR_EMIT6_TX:
1480         case INSTR_HDR_EMIT7_TX:
1481         case INSTR_HDR_EMIT8_TX:
1482                 return 1;
1483         default:
1484                 return 0;
1485         }
1486 }
1487
1488 static int
1489 instruction_is_jmp(struct instruction *instr)
1490 {
1491         switch (instr->type) {
1492         case INSTR_JMP:
1493         case INSTR_JMP_VALID:
1494         case INSTR_JMP_INVALID:
1495         case INSTR_JMP_HIT:
1496         case INSTR_JMP_MISS:
1497         case INSTR_JMP_ACTION_HIT:
1498         case INSTR_JMP_ACTION_MISS:
1499         case INSTR_JMP_EQ:
1500         case INSTR_JMP_EQ_MH:
1501         case INSTR_JMP_EQ_HM:
1502         case INSTR_JMP_EQ_HH:
1503         case INSTR_JMP_EQ_I:
1504         case INSTR_JMP_NEQ:
1505         case INSTR_JMP_NEQ_MH:
1506         case INSTR_JMP_NEQ_HM:
1507         case INSTR_JMP_NEQ_HH:
1508         case INSTR_JMP_NEQ_I:
1509         case INSTR_JMP_LT:
1510         case INSTR_JMP_LT_MH:
1511         case INSTR_JMP_LT_HM:
1512         case INSTR_JMP_LT_HH:
1513         case INSTR_JMP_LT_MI:
1514         case INSTR_JMP_LT_HI:
1515         case INSTR_JMP_GT:
1516         case INSTR_JMP_GT_MH:
1517         case INSTR_JMP_GT_HM:
1518         case INSTR_JMP_GT_HH:
1519         case INSTR_JMP_GT_MI:
1520         case INSTR_JMP_GT_HI:
1521                 return 1;
1522
1523         default:
1524                 return 0;
1525         }
1526 }
1527
1528 static int
1529 instruction_does_thread_yield(struct instruction *instr)
1530 {
1531         switch (instr->type) {
1532         case INSTR_RX:
1533         case INSTR_TABLE:
1534         case INSTR_TABLE_AF:
1535         case INSTR_SELECTOR:
1536         case INSTR_LEARNER:
1537         case INSTR_LEARNER_AF:
1538         case INSTR_EXTERN_OBJ:
1539         case INSTR_EXTERN_FUNC:
1540                 return 1;
1541         default:
1542                 return 0;
1543         }
1544 }
1545
1546 static struct field *
1547 action_field_parse(struct action *action, const char *name);
1548
1549 static struct field *
1550 struct_field_parse(struct rte_swx_pipeline *p,
1551                    struct action *action,
1552                    const char *name,
1553                    uint32_t *struct_id)
1554 {
1555         struct field *f;
1556
1557         switch (name[0]) {
1558         case 'h':
1559         {
1560                 struct header *header;
1561
1562                 f = header_field_parse(p, name, &header);
1563                 if (!f)
1564                         return NULL;
1565
1566                 *struct_id = header->struct_id;
1567                 return f;
1568         }
1569
1570         case 'm':
1571         {
1572                 f = metadata_field_parse(p, name);
1573                 if (!f)
1574                         return NULL;
1575
1576                 *struct_id = p->metadata_struct_id;
1577                 return f;
1578         }
1579
1580         case 't':
1581         {
1582                 if (!action)
1583                         return NULL;
1584
1585                 f = action_field_parse(action, name);
1586                 if (!f)
1587                         return NULL;
1588
1589                 *struct_id = 0;
1590                 return f;
1591         }
1592
1593         case 'e':
1594         {
1595                 struct extern_obj *obj;
1596
1597                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1598                 if (!f)
1599                         return NULL;
1600
1601                 *struct_id = obj->struct_id;
1602                 return f;
1603         }
1604
1605         case 'f':
1606         {
1607                 struct extern_func *func;
1608
1609                 f = extern_func_mailbox_field_parse(p, name, &func);
1610                 if (!f)
1611                         return NULL;
1612
1613                 *struct_id = func->struct_id;
1614                 return f;
1615         }
1616
1617         default:
1618                 return NULL;
1619         }
1620 }
1621
1622 /*
1623  * rx.
1624  */
1625 static int
1626 instr_rx_translate(struct rte_swx_pipeline *p,
1627                    struct action *action,
1628                    char **tokens,
1629                    int n_tokens,
1630                    struct instruction *instr,
1631                    struct instruction_data *data __rte_unused)
1632 {
1633         struct field *f;
1634
1635         CHECK(!action, EINVAL);
1636         CHECK(n_tokens == 2, EINVAL);
1637
1638         f = metadata_field_parse(p, tokens[1]);
1639         CHECK(f, EINVAL);
1640
1641         instr->type = INSTR_RX;
1642         instr->io.io.offset = f->offset / 8;
1643         instr->io.io.n_bits = f->n_bits;
1644         return 0;
1645 }
1646
1647 /*
1648  * tx.
1649  */
1650 static int
1651 instr_tx_translate(struct rte_swx_pipeline *p,
1652                    struct action *action __rte_unused,
1653                    char **tokens,
1654                    int n_tokens,
1655                    struct instruction *instr,
1656                    struct instruction_data *data __rte_unused)
1657 {
1658         char *port = tokens[1];
1659         struct field *f;
1660         uint32_t port_val;
1661
1662         CHECK(n_tokens == 2, EINVAL);
1663
1664         f = metadata_field_parse(p, port);
1665         if (f) {
1666                 instr->type = INSTR_TX;
1667                 instr->io.io.offset = f->offset / 8;
1668                 instr->io.io.n_bits = f->n_bits;
1669                 return 0;
1670         }
1671
1672         /* TX_I. */
1673         port_val = strtoul(port, &port, 0);
1674         CHECK(!port[0], EINVAL);
1675
1676         instr->type = INSTR_TX_I;
1677         instr->io.io.val = port_val;
1678         return 0;
1679 }
1680
1681 static int
1682 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1683                      struct action *action __rte_unused,
1684                      char **tokens __rte_unused,
1685                      int n_tokens,
1686                      struct instruction *instr,
1687                      struct instruction_data *data __rte_unused)
1688 {
1689         CHECK(n_tokens == 1, EINVAL);
1690
1691         /* DROP. */
1692         instr->type = INSTR_DROP;
1693         return 0;
1694 }
1695
1696 static inline void
1697 instr_tx_exec(struct rte_swx_pipeline *p)
1698 {
1699         struct thread *t = &p->threads[p->thread_id];
1700         struct instruction *ip = t->ip;
1701
1702         __instr_tx_exec(p, t, ip);
1703
1704         /* Thread. */
1705         thread_ip_reset(p, t);
1706         instr_rx_exec(p);
1707 }
1708
1709 static inline void
1710 instr_tx_i_exec(struct rte_swx_pipeline *p)
1711 {
1712         struct thread *t = &p->threads[p->thread_id];
1713         struct instruction *ip = t->ip;
1714
1715         __instr_tx_i_exec(p, t, ip);
1716
1717         /* Thread. */
1718         thread_ip_reset(p, t);
1719         instr_rx_exec(p);
1720 }
1721
1722 static inline void
1723 instr_drop_exec(struct rte_swx_pipeline *p)
1724 {
1725         struct thread *t = &p->threads[p->thread_id];
1726         struct instruction *ip = t->ip;
1727
1728         __instr_drop_exec(p, t, ip);
1729
1730         /* Thread. */
1731         thread_ip_reset(p, t);
1732         instr_rx_exec(p);
1733 }
1734
1735 /*
1736  * mirror.
1737  */
1738 static int
1739 instr_mirror_translate(struct rte_swx_pipeline *p,
1740                        struct action *action,
1741                        char **tokens,
1742                        int n_tokens,
1743                        struct instruction *instr,
1744                        struct instruction_data *data __rte_unused)
1745 {
1746         char *dst = tokens[1], *src = tokens[2];
1747         struct field *fdst, *fsrc;
1748         uint32_t dst_struct_id = 0, src_struct_id = 0;
1749
1750         CHECK(n_tokens == 3, EINVAL);
1751
1752         fdst = struct_field_parse(p, action, dst, &dst_struct_id);
1753         CHECK(fdst, EINVAL);
1754         CHECK(dst[0] != 'h', EINVAL);
1755         CHECK(!fdst->var_size, EINVAL);
1756
1757         fsrc = struct_field_parse(p, action, src, &src_struct_id);
1758         CHECK(fsrc, EINVAL);
1759         CHECK(src[0] != 'h', EINVAL);
1760         CHECK(!fsrc->var_size, EINVAL);
1761
1762         instr->type = INSTR_MIRROR;
1763         instr->mirror.dst.struct_id = (uint8_t)dst_struct_id;
1764         instr->mirror.dst.n_bits = fdst->n_bits;
1765         instr->mirror.dst.offset = fdst->offset / 8;
1766         instr->mirror.src.struct_id = (uint8_t)src_struct_id;
1767         instr->mirror.src.n_bits = fsrc->n_bits;
1768         instr->mirror.src.offset = fsrc->offset / 8;
1769
1770         return 0;
1771 }
1772
1773 static inline void
1774 instr_mirror_exec(struct rte_swx_pipeline *p)
1775 {
1776         struct thread *t = &p->threads[p->thread_id];
1777         struct instruction *ip = t->ip;
1778
1779         __instr_mirror_exec(p, t, ip);
1780
1781         /* Thread. */
1782         thread_ip_inc(p);
1783 }
1784
1785 /*
1786  * recirculate.
1787  */
1788 static int
1789 instr_recirculate_translate(struct rte_swx_pipeline *p __rte_unused,
1790                             struct action *action __rte_unused,
1791                             char **tokens __rte_unused,
1792                             int n_tokens,
1793                             struct instruction *instr,
1794                             struct instruction_data *data __rte_unused)
1795 {
1796         CHECK(n_tokens == 1, EINVAL);
1797
1798         instr->type = INSTR_RECIRCULATE;
1799         return 0;
1800 }
1801
1802 static int
1803 instr_recircid_translate(struct rte_swx_pipeline *p,
1804                          struct action *action __rte_unused,
1805                          char **tokens,
1806                          int n_tokens,
1807                          struct instruction *instr,
1808                          struct instruction_data *data __rte_unused)
1809 {
1810         struct field *f;
1811
1812         CHECK(n_tokens == 2, EINVAL);
1813
1814         f = metadata_field_parse(p, tokens[1]);
1815         CHECK(f, EINVAL);
1816
1817         instr->type = INSTR_RECIRCID;
1818         instr->io.io.offset = f->offset / 8;
1819         instr->io.io.n_bits = f->n_bits;
1820         return 0;
1821 }
1822
1823 static inline void
1824 instr_recirculate_exec(struct rte_swx_pipeline *p)
1825 {
1826         struct thread *t = &p->threads[p->thread_id];
1827         struct instruction *ip = t->ip;
1828
1829         __instr_recirculate_exec(p, t, ip);
1830
1831         /* Thread. */
1832         thread_ip_inc(p);
1833 }
1834
1835 static inline void
1836 instr_recircid_exec(struct rte_swx_pipeline *p)
1837 {
1838         struct thread *t = &p->threads[p->thread_id];
1839         struct instruction *ip = t->ip;
1840
1841         __instr_recircid_exec(p, t, ip);
1842
1843         /* Thread. */
1844         thread_ip_inc(p);
1845 }
1846
1847 /*
1848  * extract.
1849  */
1850 static int
1851 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1852                             struct action *action,
1853                             char **tokens,
1854                             int n_tokens,
1855                             struct instruction *instr,
1856                             struct instruction_data *data __rte_unused)
1857 {
1858         struct header *h;
1859
1860         CHECK(!action, EINVAL);
1861         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1862
1863         h = header_parse(p, tokens[1]);
1864         CHECK(h, EINVAL);
1865
1866         if (n_tokens == 2) {
1867                 CHECK(!h->st->var_size, EINVAL);
1868
1869                 instr->type = INSTR_HDR_EXTRACT;
1870                 instr->io.hdr.header_id[0] = h->id;
1871                 instr->io.hdr.struct_id[0] = h->struct_id;
1872                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1873         } else {
1874                 struct field *mf;
1875
1876                 CHECK(h->st->var_size, EINVAL);
1877
1878                 mf = metadata_field_parse(p, tokens[2]);
1879                 CHECK(mf, EINVAL);
1880                 CHECK(!mf->var_size, EINVAL);
1881
1882                 instr->type = INSTR_HDR_EXTRACT_M;
1883                 instr->io.io.offset = mf->offset / 8;
1884                 instr->io.io.n_bits = mf->n_bits;
1885                 instr->io.hdr.header_id[0] = h->id;
1886                 instr->io.hdr.struct_id[0] = h->struct_id;
1887                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1888         }
1889
1890         return 0;
1891 }
1892
1893 static int
1894 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1895                               struct action *action,
1896                               char **tokens,
1897                               int n_tokens,
1898                               struct instruction *instr,
1899                               struct instruction_data *data __rte_unused)
1900 {
1901         struct header *h;
1902
1903         CHECK(!action, EINVAL);
1904         CHECK(n_tokens == 2, EINVAL);
1905
1906         h = header_parse(p, tokens[1]);
1907         CHECK(h, EINVAL);
1908         CHECK(!h->st->var_size, EINVAL);
1909
1910         instr->type = INSTR_HDR_LOOKAHEAD;
1911         instr->io.hdr.header_id[0] = h->id;
1912         instr->io.hdr.struct_id[0] = h->struct_id;
1913         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1914
1915         return 0;
1916 }
1917
1918 static inline void
1919 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1920 {
1921         struct thread *t = &p->threads[p->thread_id];
1922         struct instruction *ip = t->ip;
1923
1924         __instr_hdr_extract_exec(p, t, ip);
1925
1926         /* Thread. */
1927         thread_ip_inc(p);
1928 }
1929
1930 static inline void
1931 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1932 {
1933         struct thread *t = &p->threads[p->thread_id];
1934         struct instruction *ip = t->ip;
1935
1936         __instr_hdr_extract2_exec(p, t, ip);
1937
1938         /* Thread. */
1939         thread_ip_inc(p);
1940 }
1941
1942 static inline void
1943 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1944 {
1945         struct thread *t = &p->threads[p->thread_id];
1946         struct instruction *ip = t->ip;
1947
1948         __instr_hdr_extract3_exec(p, t, ip);
1949
1950         /* Thread. */
1951         thread_ip_inc(p);
1952 }
1953
1954 static inline void
1955 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1956 {
1957         struct thread *t = &p->threads[p->thread_id];
1958         struct instruction *ip = t->ip;
1959
1960         __instr_hdr_extract4_exec(p, t, ip);
1961
1962         /* Thread. */
1963         thread_ip_inc(p);
1964 }
1965
1966 static inline void
1967 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1968 {
1969         struct thread *t = &p->threads[p->thread_id];
1970         struct instruction *ip = t->ip;
1971
1972         __instr_hdr_extract5_exec(p, t, ip);
1973
1974         /* Thread. */
1975         thread_ip_inc(p);
1976 }
1977
1978 static inline void
1979 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1980 {
1981         struct thread *t = &p->threads[p->thread_id];
1982         struct instruction *ip = t->ip;
1983
1984         __instr_hdr_extract6_exec(p, t, ip);
1985
1986         /* Thread. */
1987         thread_ip_inc(p);
1988 }
1989
1990 static inline void
1991 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1992 {
1993         struct thread *t = &p->threads[p->thread_id];
1994         struct instruction *ip = t->ip;
1995
1996         __instr_hdr_extract7_exec(p, t, ip);
1997
1998         /* Thread. */
1999         thread_ip_inc(p);
2000 }
2001
2002 static inline void
2003 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2004 {
2005         struct thread *t = &p->threads[p->thread_id];
2006         struct instruction *ip = t->ip;
2007
2008         __instr_hdr_extract8_exec(p, t, ip);
2009
2010         /* Thread. */
2011         thread_ip_inc(p);
2012 }
2013
2014 static inline void
2015 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
2016 {
2017         struct thread *t = &p->threads[p->thread_id];
2018         struct instruction *ip = t->ip;
2019
2020         __instr_hdr_extract_m_exec(p, t, ip);
2021
2022         /* Thread. */
2023         thread_ip_inc(p);
2024 }
2025
2026 static inline void
2027 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
2028 {
2029         struct thread *t = &p->threads[p->thread_id];
2030         struct instruction *ip = t->ip;
2031
2032         __instr_hdr_lookahead_exec(p, t, ip);
2033
2034         /* Thread. */
2035         thread_ip_inc(p);
2036 }
2037
2038 /*
2039  * emit.
2040  */
2041 static int
2042 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2043                          struct action *action __rte_unused,
2044                          char **tokens,
2045                          int n_tokens,
2046                          struct instruction *instr,
2047                          struct instruction_data *data __rte_unused)
2048 {
2049         struct header *h;
2050
2051         CHECK(n_tokens == 2, EINVAL);
2052
2053         h = header_parse(p, tokens[1]);
2054         CHECK(h, EINVAL);
2055
2056         instr->type = INSTR_HDR_EMIT;
2057         instr->io.hdr.header_id[0] = h->id;
2058         instr->io.hdr.struct_id[0] = h->struct_id;
2059         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2060         return 0;
2061 }
2062
2063 static inline void
2064 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2065 {
2066         struct thread *t = &p->threads[p->thread_id];
2067         struct instruction *ip = t->ip;
2068
2069         __instr_hdr_emit_exec(p, t, ip);
2070
2071         /* Thread. */
2072         thread_ip_inc(p);
2073 }
2074
2075 static inline void
2076 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2077 {
2078         struct thread *t = &p->threads[p->thread_id];
2079         struct instruction *ip = t->ip;
2080
2081         __instr_hdr_emit_tx_exec(p, t, ip);
2082
2083         /* Thread. */
2084         thread_ip_reset(p, t);
2085         instr_rx_exec(p);
2086 }
2087
2088 static inline void
2089 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2090 {
2091         struct thread *t = &p->threads[p->thread_id];
2092         struct instruction *ip = t->ip;
2093
2094         __instr_hdr_emit2_tx_exec(p, t, ip);
2095
2096         /* Thread. */
2097         thread_ip_reset(p, t);
2098         instr_rx_exec(p);
2099 }
2100
2101 static inline void
2102 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2103 {
2104         struct thread *t = &p->threads[p->thread_id];
2105         struct instruction *ip = t->ip;
2106
2107         __instr_hdr_emit3_tx_exec(p, t, ip);
2108
2109         /* Thread. */
2110         thread_ip_reset(p, t);
2111         instr_rx_exec(p);
2112 }
2113
2114 static inline void
2115 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2116 {
2117         struct thread *t = &p->threads[p->thread_id];
2118         struct instruction *ip = t->ip;
2119
2120         __instr_hdr_emit4_tx_exec(p, t, ip);
2121
2122         /* Thread. */
2123         thread_ip_reset(p, t);
2124         instr_rx_exec(p);
2125 }
2126
2127 static inline void
2128 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2129 {
2130         struct thread *t = &p->threads[p->thread_id];
2131         struct instruction *ip = t->ip;
2132
2133         __instr_hdr_emit5_tx_exec(p, t, ip);
2134
2135         /* Thread. */
2136         thread_ip_reset(p, t);
2137         instr_rx_exec(p);
2138 }
2139
2140 static inline void
2141 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2142 {
2143         struct thread *t = &p->threads[p->thread_id];
2144         struct instruction *ip = t->ip;
2145
2146         __instr_hdr_emit6_tx_exec(p, t, ip);
2147
2148         /* Thread. */
2149         thread_ip_reset(p, t);
2150         instr_rx_exec(p);
2151 }
2152
2153 static inline void
2154 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2155 {
2156         struct thread *t = &p->threads[p->thread_id];
2157         struct instruction *ip = t->ip;
2158
2159         __instr_hdr_emit7_tx_exec(p, t, ip);
2160
2161         /* Thread. */
2162         thread_ip_reset(p, t);
2163         instr_rx_exec(p);
2164 }
2165
2166 static inline void
2167 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2168 {
2169         struct thread *t = &p->threads[p->thread_id];
2170         struct instruction *ip = t->ip;
2171
2172         __instr_hdr_emit8_tx_exec(p, t, ip);
2173
2174         /* Thread. */
2175         thread_ip_reset(p, t);
2176         instr_rx_exec(p);
2177 }
2178
2179 /*
2180  * validate.
2181  */
2182 static int
2183 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2184                              struct action *action __rte_unused,
2185                              char **tokens,
2186                              int n_tokens,
2187                              struct instruction *instr,
2188                              struct instruction_data *data __rte_unused)
2189 {
2190         struct header *h;
2191
2192         CHECK(n_tokens == 2, EINVAL);
2193
2194         h = header_parse(p, tokens[1]);
2195         CHECK(h, EINVAL);
2196
2197         instr->type = INSTR_HDR_VALIDATE;
2198         instr->valid.header_id = h->id;
2199         return 0;
2200 }
2201
2202 static inline void
2203 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2204 {
2205         struct thread *t = &p->threads[p->thread_id];
2206         struct instruction *ip = t->ip;
2207
2208         __instr_hdr_validate_exec(p, t, ip);
2209
2210         /* Thread. */
2211         thread_ip_inc(p);
2212 }
2213
2214 /*
2215  * invalidate.
2216  */
2217 static int
2218 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2219                                struct action *action __rte_unused,
2220                                char **tokens,
2221                                int n_tokens,
2222                                struct instruction *instr,
2223                                struct instruction_data *data __rte_unused)
2224 {
2225         struct header *h;
2226
2227         CHECK(n_tokens == 2, EINVAL);
2228
2229         h = header_parse(p, tokens[1]);
2230         CHECK(h, EINVAL);
2231
2232         instr->type = INSTR_HDR_INVALIDATE;
2233         instr->valid.header_id = h->id;
2234         return 0;
2235 }
2236
2237 static inline void
2238 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2239 {
2240         struct thread *t = &p->threads[p->thread_id];
2241         struct instruction *ip = t->ip;
2242
2243         __instr_hdr_invalidate_exec(p, t, ip);
2244
2245         /* Thread. */
2246         thread_ip_inc(p);
2247 }
2248
2249 /*
2250  * table.
2251  */
2252 static struct table *
2253 table_find(struct rte_swx_pipeline *p, const char *name);
2254
2255 static struct selector *
2256 selector_find(struct rte_swx_pipeline *p, const char *name);
2257
2258 static struct learner *
2259 learner_find(struct rte_swx_pipeline *p, const char *name);
2260
2261 static int
2262 instr_table_translate(struct rte_swx_pipeline *p,
2263                       struct action *action,
2264                       char **tokens,
2265                       int n_tokens,
2266                       struct instruction *instr,
2267                       struct instruction_data *data __rte_unused)
2268 {
2269         struct table *t;
2270         struct selector *s;
2271         struct learner *l;
2272
2273         CHECK(!action, EINVAL);
2274         CHECK(n_tokens == 2, EINVAL);
2275
2276         t = table_find(p, tokens[1]);
2277         if (t) {
2278                 instr->type = INSTR_TABLE;
2279                 instr->table.table_id = t->id;
2280                 return 0;
2281         }
2282
2283         s = selector_find(p, tokens[1]);
2284         if (s) {
2285                 instr->type = INSTR_SELECTOR;
2286                 instr->table.table_id = s->id;
2287                 return 0;
2288         }
2289
2290         l = learner_find(p, tokens[1]);
2291         if (l) {
2292                 instr->type = INSTR_LEARNER;
2293                 instr->table.table_id = l->id;
2294                 return 0;
2295         }
2296
2297         CHECK(0, EINVAL);
2298 }
2299
2300 static inline void
2301 instr_table_exec(struct rte_swx_pipeline *p)
2302 {
2303         struct thread *t = &p->threads[p->thread_id];
2304         struct instruction *ip = t->ip;
2305         uint32_t table_id = ip->table.table_id;
2306         struct rte_swx_table_state *ts = &t->table_state[table_id];
2307         struct table_runtime *table = &t->tables[table_id];
2308         struct table_statistics *stats = &p->table_stats[table_id];
2309         uint64_t action_id, n_pkts_hit, n_pkts_action;
2310         uint8_t *action_data;
2311         int done, hit;
2312
2313         /* Table. */
2314         done = table->func(ts->obj,
2315                            table->mailbox,
2316                            table->key,
2317                            &action_id,
2318                            &action_data,
2319                            &hit);
2320         if (!done) {
2321                 /* Thread. */
2322                 TRACE("[Thread %2u] table %u (not finalized)\n",
2323                       p->thread_id,
2324                       table_id);
2325
2326                 thread_yield(p);
2327                 return;
2328         }
2329
2330         action_id = hit ? action_id : ts->default_action_id;
2331         action_data = hit ? action_data : ts->default_action_data;
2332         n_pkts_hit = stats->n_pkts_hit[hit];
2333         n_pkts_action = stats->n_pkts_action[action_id];
2334
2335         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2336               p->thread_id,
2337               table_id,
2338               hit ? "hit" : "miss",
2339               (uint32_t)action_id);
2340
2341         t->action_id = action_id;
2342         t->structs[0] = action_data;
2343         t->hit = hit;
2344         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2345         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2346
2347         /* Thread. */
2348         thread_ip_action_call(p, t, action_id);
2349 }
2350
2351 static inline void
2352 instr_table_af_exec(struct rte_swx_pipeline *p)
2353 {
2354         struct thread *t = &p->threads[p->thread_id];
2355         struct instruction *ip = t->ip;
2356         uint32_t table_id = ip->table.table_id;
2357         struct rte_swx_table_state *ts = &t->table_state[table_id];
2358         struct table_runtime *table = &t->tables[table_id];
2359         struct table_statistics *stats = &p->table_stats[table_id];
2360         uint64_t action_id, n_pkts_hit, n_pkts_action;
2361         uint8_t *action_data;
2362         action_func_t action_func;
2363         int done, hit;
2364
2365         /* Table. */
2366         done = table->func(ts->obj,
2367                            table->mailbox,
2368                            table->key,
2369                            &action_id,
2370                            &action_data,
2371                            &hit);
2372         if (!done) {
2373                 /* Thread. */
2374                 TRACE("[Thread %2u] table %u (not finalized)\n",
2375                       p->thread_id,
2376                       table_id);
2377
2378                 thread_yield(p);
2379                 return;
2380         }
2381
2382         action_id = hit ? action_id : ts->default_action_id;
2383         action_data = hit ? action_data : ts->default_action_data;
2384         action_func = p->action_funcs[action_id];
2385         n_pkts_hit = stats->n_pkts_hit[hit];
2386         n_pkts_action = stats->n_pkts_action[action_id];
2387
2388         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2389               p->thread_id,
2390               table_id,
2391               hit ? "hit" : "miss",
2392               (uint32_t)action_id);
2393
2394         t->action_id = action_id;
2395         t->structs[0] = action_data;
2396         t->hit = hit;
2397         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2398         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2399
2400         /* Thread. */
2401         thread_ip_inc(p);
2402
2403         /* Action. */
2404         action_func(p);
2405 }
2406
2407 static inline void
2408 instr_selector_exec(struct rte_swx_pipeline *p)
2409 {
2410         struct thread *t = &p->threads[p->thread_id];
2411         struct instruction *ip = t->ip;
2412         uint32_t selector_id = ip->table.table_id;
2413         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2414         struct selector_runtime *selector = &t->selectors[selector_id];
2415         struct selector_statistics *stats = &p->selector_stats[selector_id];
2416         uint64_t n_pkts = stats->n_pkts;
2417         int done;
2418
2419         /* Table. */
2420         done = rte_swx_table_selector_select(ts->obj,
2421                            selector->mailbox,
2422                            selector->group_id_buffer,
2423                            selector->selector_buffer,
2424                            selector->member_id_buffer);
2425         if (!done) {
2426                 /* Thread. */
2427                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2428                       p->thread_id,
2429                       selector_id);
2430
2431                 thread_yield(p);
2432                 return;
2433         }
2434
2435
2436         TRACE("[Thread %2u] selector %u\n",
2437               p->thread_id,
2438               selector_id);
2439
2440         stats->n_pkts = n_pkts + 1;
2441
2442         /* Thread. */
2443         thread_ip_inc(p);
2444 }
2445
2446 static inline void
2447 instr_learner_exec(struct rte_swx_pipeline *p)
2448 {
2449         struct thread *t = &p->threads[p->thread_id];
2450         struct instruction *ip = t->ip;
2451         uint32_t learner_id = ip->table.table_id;
2452         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2453                 p->n_selectors + learner_id];
2454         struct learner_runtime *l = &t->learners[learner_id];
2455         struct learner_statistics *stats = &p->learner_stats[learner_id];
2456         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2457         uint8_t *action_data;
2458         int done, hit;
2459
2460         /* Table. */
2461         time = rte_get_tsc_cycles();
2462
2463         done = rte_swx_table_learner_lookup(ts->obj,
2464                                             l->mailbox,
2465                                             time,
2466                                             l->key,
2467                                             &action_id,
2468                                             &action_data,
2469                                             &hit);
2470         if (!done) {
2471                 /* Thread. */
2472                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2473                       p->thread_id,
2474                       learner_id);
2475
2476                 thread_yield(p);
2477                 return;
2478         }
2479
2480         action_id = hit ? action_id : ts->default_action_id;
2481         action_data = hit ? action_data : ts->default_action_data;
2482         n_pkts_hit = stats->n_pkts_hit[hit];
2483         n_pkts_action = stats->n_pkts_action[action_id];
2484
2485         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2486               p->thread_id,
2487               learner_id,
2488               hit ? "hit" : "miss",
2489               (uint32_t)action_id);
2490
2491         t->action_id = action_id;
2492         t->structs[0] = action_data;
2493         t->hit = hit;
2494         t->learner_id = learner_id;
2495         t->time = time;
2496         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2497         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2498
2499         /* Thread. */
2500         thread_ip_action_call(p, t, action_id);
2501 }
2502
2503 static inline void
2504 instr_learner_af_exec(struct rte_swx_pipeline *p)
2505 {
2506         struct thread *t = &p->threads[p->thread_id];
2507         struct instruction *ip = t->ip;
2508         uint32_t learner_id = ip->table.table_id;
2509         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2510                 p->n_selectors + learner_id];
2511         struct learner_runtime *l = &t->learners[learner_id];
2512         struct learner_statistics *stats = &p->learner_stats[learner_id];
2513         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2514         uint8_t *action_data;
2515         action_func_t action_func;
2516         int done, hit;
2517
2518         /* Table. */
2519         time = rte_get_tsc_cycles();
2520
2521         done = rte_swx_table_learner_lookup(ts->obj,
2522                                             l->mailbox,
2523                                             time,
2524                                             l->key,
2525                                             &action_id,
2526                                             &action_data,
2527                                             &hit);
2528         if (!done) {
2529                 /* Thread. */
2530                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2531                       p->thread_id,
2532                       learner_id);
2533
2534                 thread_yield(p);
2535                 return;
2536         }
2537
2538         action_id = hit ? action_id : ts->default_action_id;
2539         action_data = hit ? action_data : ts->default_action_data;
2540         action_func = p->action_funcs[action_id];
2541         n_pkts_hit = stats->n_pkts_hit[hit];
2542         n_pkts_action = stats->n_pkts_action[action_id];
2543
2544         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2545               p->thread_id,
2546               learner_id,
2547               hit ? "hit" : "miss",
2548               (uint32_t)action_id);
2549
2550         t->action_id = action_id;
2551         t->structs[0] = action_data;
2552         t->hit = hit;
2553         t->learner_id = learner_id;
2554         t->time = time;
2555         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2556         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2557
2558         /* Thread. */
2559         thread_ip_action_call(p, t, action_id);
2560
2561         /* Action */
2562         action_func(p);
2563 }
2564
2565 /*
2566  * learn.
2567  */
2568 static struct action *
2569 action_find(struct rte_swx_pipeline *p, const char *name);
2570
2571 static int
2572 action_has_nbo_args(struct action *a);
2573
2574 static int
2575 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2576
2577 static int
2578 instr_learn_translate(struct rte_swx_pipeline *p,
2579                       struct action *action,
2580                       char **tokens,
2581                       int n_tokens,
2582                       struct instruction *instr,
2583                       struct instruction_data *data __rte_unused)
2584 {
2585         struct action *a;
2586         const char *mf_name;
2587         uint32_t mf_offset = 0;
2588
2589         CHECK(action, EINVAL);
2590         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2591
2592         a = action_find(p, tokens[1]);
2593         CHECK(a, EINVAL);
2594         CHECK(!action_has_nbo_args(a), EINVAL);
2595
2596         mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2597         CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2598
2599         if (mf_name) {
2600                 struct field *mf;
2601
2602                 mf = metadata_field_parse(p, mf_name);
2603                 CHECK(mf, EINVAL);
2604
2605                 mf_offset = mf->offset / 8;
2606         }
2607
2608         instr->type = INSTR_LEARNER_LEARN;
2609         instr->learn.action_id = a->id;
2610         instr->learn.mf_offset = mf_offset;
2611
2612         return 0;
2613 }
2614
2615 static inline void
2616 instr_learn_exec(struct rte_swx_pipeline *p)
2617 {
2618         struct thread *t = &p->threads[p->thread_id];
2619         struct instruction *ip = t->ip;
2620
2621         __instr_learn_exec(p, t, ip);
2622
2623         /* Thread. */
2624         thread_ip_inc(p);
2625 }
2626
2627 /*
2628  * forget.
2629  */
2630 static int
2631 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2632                        struct action *action,
2633                        char **tokens __rte_unused,
2634                        int n_tokens,
2635                        struct instruction *instr,
2636                        struct instruction_data *data __rte_unused)
2637 {
2638         CHECK(action, EINVAL);
2639         CHECK(n_tokens == 1, EINVAL);
2640
2641         instr->type = INSTR_LEARNER_FORGET;
2642
2643         return 0;
2644 }
2645
2646 static inline void
2647 instr_forget_exec(struct rte_swx_pipeline *p)
2648 {
2649         struct thread *t = &p->threads[p->thread_id];
2650         struct instruction *ip = t->ip;
2651
2652         __instr_forget_exec(p, t, ip);
2653
2654         /* Thread. */
2655         thread_ip_inc(p);
2656 }
2657
2658 /*
2659  * extern.
2660  */
2661 static int
2662 instr_extern_translate(struct rte_swx_pipeline *p,
2663                        struct action *action __rte_unused,
2664                        char **tokens,
2665                        int n_tokens,
2666                        struct instruction *instr,
2667                        struct instruction_data *data __rte_unused)
2668 {
2669         char *token = tokens[1];
2670
2671         CHECK(n_tokens == 2, EINVAL);
2672
2673         if (token[0] == 'e') {
2674                 struct extern_obj *obj;
2675                 struct extern_type_member_func *func;
2676
2677                 func = extern_obj_member_func_parse(p, token, &obj);
2678                 CHECK(func, EINVAL);
2679
2680                 instr->type = INSTR_EXTERN_OBJ;
2681                 instr->ext_obj.ext_obj_id = obj->id;
2682                 instr->ext_obj.func_id = func->id;
2683
2684                 return 0;
2685         }
2686
2687         if (token[0] == 'f') {
2688                 struct extern_func *func;
2689
2690                 func = extern_func_parse(p, token);
2691                 CHECK(func, EINVAL);
2692
2693                 instr->type = INSTR_EXTERN_FUNC;
2694                 instr->ext_func.ext_func_id = func->id;
2695
2696                 return 0;
2697         }
2698
2699         CHECK(0, EINVAL);
2700 }
2701
2702 static inline void
2703 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2704 {
2705         struct thread *t = &p->threads[p->thread_id];
2706         struct instruction *ip = t->ip;
2707         uint32_t done;
2708
2709         /* Extern object member function execute. */
2710         done = __instr_extern_obj_exec(p, t, ip);
2711
2712         /* Thread. */
2713         thread_ip_inc_cond(t, done);
2714         thread_yield_cond(p, done ^ 1);
2715 }
2716
2717 static inline void
2718 instr_extern_func_exec(struct rte_swx_pipeline *p)
2719 {
2720         struct thread *t = &p->threads[p->thread_id];
2721         struct instruction *ip = t->ip;
2722         uint32_t done;
2723
2724         /* Extern function execute. */
2725         done = __instr_extern_func_exec(p, t, ip);
2726
2727         /* Thread. */
2728         thread_ip_inc_cond(t, done);
2729         thread_yield_cond(p, done ^ 1);
2730 }
2731
2732 /*
2733  * mov.
2734  */
2735 static int
2736 instr_mov_translate(struct rte_swx_pipeline *p,
2737                     struct action *action,
2738                     char **tokens,
2739                     int n_tokens,
2740                     struct instruction *instr,
2741                     struct instruction_data *data __rte_unused)
2742 {
2743         char *dst = tokens[1], *src = tokens[2];
2744         struct field *fdst, *fsrc;
2745         uint64_t src_val;
2746         uint32_t dst_struct_id = 0, src_struct_id = 0;
2747
2748         CHECK(n_tokens == 3, EINVAL);
2749
2750         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2751         CHECK(fdst, EINVAL);
2752         CHECK(!fdst->var_size, EINVAL);
2753
2754         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2755         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2756         if (fsrc) {
2757                 CHECK(!fsrc->var_size, EINVAL);
2758
2759                 instr->type = INSTR_MOV;
2760                 if (dst[0] != 'h' && src[0] == 'h')
2761                         instr->type = INSTR_MOV_MH;
2762                 if (dst[0] == 'h' && src[0] != 'h')
2763                         instr->type = INSTR_MOV_HM;
2764                 if (dst[0] == 'h' && src[0] == 'h')
2765                         instr->type = INSTR_MOV_HH;
2766
2767                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2768                 instr->mov.dst.n_bits = fdst->n_bits;
2769                 instr->mov.dst.offset = fdst->offset / 8;
2770                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2771                 instr->mov.src.n_bits = fsrc->n_bits;
2772                 instr->mov.src.offset = fsrc->offset / 8;
2773                 return 0;
2774         }
2775
2776         /* MOV_I. */
2777         src_val = strtoull(src, &src, 0);
2778         CHECK(!src[0], EINVAL);
2779
2780         if (dst[0] == 'h')
2781                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2782
2783         instr->type = INSTR_MOV_I;
2784         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2785         instr->mov.dst.n_bits = fdst->n_bits;
2786         instr->mov.dst.offset = fdst->offset / 8;
2787         instr->mov.src_val = src_val;
2788         return 0;
2789 }
2790
2791 static inline void
2792 instr_mov_exec(struct rte_swx_pipeline *p)
2793 {
2794         struct thread *t = &p->threads[p->thread_id];
2795         struct instruction *ip = t->ip;
2796
2797         __instr_mov_exec(p, t, ip);
2798
2799         /* Thread. */
2800         thread_ip_inc(p);
2801 }
2802
2803 static inline void
2804 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2805 {
2806         struct thread *t = &p->threads[p->thread_id];
2807         struct instruction *ip = t->ip;
2808
2809         __instr_mov_mh_exec(p, t, ip);
2810
2811         /* Thread. */
2812         thread_ip_inc(p);
2813 }
2814
2815 static inline void
2816 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2817 {
2818         struct thread *t = &p->threads[p->thread_id];
2819         struct instruction *ip = t->ip;
2820
2821         __instr_mov_hm_exec(p, t, ip);
2822
2823         /* Thread. */
2824         thread_ip_inc(p);
2825 }
2826
2827 static inline void
2828 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2829 {
2830         struct thread *t = &p->threads[p->thread_id];
2831         struct instruction *ip = t->ip;
2832
2833         __instr_mov_hh_exec(p, t, ip);
2834
2835         /* Thread. */
2836         thread_ip_inc(p);
2837 }
2838
2839 static inline void
2840 instr_mov_i_exec(struct rte_swx_pipeline *p)
2841 {
2842         struct thread *t = &p->threads[p->thread_id];
2843         struct instruction *ip = t->ip;
2844
2845         __instr_mov_i_exec(p, t, ip);
2846
2847         /* Thread. */
2848         thread_ip_inc(p);
2849 }
2850
2851 /*
2852  * dma.
2853  */
2854 static inline void
2855 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2856 {
2857         struct thread *t = &p->threads[p->thread_id];
2858         struct instruction *ip = t->ip;
2859
2860         __instr_dma_ht_exec(p, t, ip);
2861
2862         /* Thread. */
2863         thread_ip_inc(p);
2864 }
2865
2866 static inline void
2867 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2868 {
2869         struct thread *t = &p->threads[p->thread_id];
2870         struct instruction *ip = t->ip;
2871
2872         __instr_dma_ht2_exec(p, t, ip);
2873
2874         /* Thread. */
2875         thread_ip_inc(p);
2876 }
2877
2878 static inline void
2879 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2880 {
2881         struct thread *t = &p->threads[p->thread_id];
2882         struct instruction *ip = t->ip;
2883
2884         __instr_dma_ht3_exec(p, t, ip);
2885
2886         /* Thread. */
2887         thread_ip_inc(p);
2888 }
2889
2890 static inline void
2891 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2892 {
2893         struct thread *t = &p->threads[p->thread_id];
2894         struct instruction *ip = t->ip;
2895
2896         __instr_dma_ht4_exec(p, t, ip);
2897
2898         /* Thread. */
2899         thread_ip_inc(p);
2900 }
2901
2902 static inline void
2903 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2904 {
2905         struct thread *t = &p->threads[p->thread_id];
2906         struct instruction *ip = t->ip;
2907
2908         __instr_dma_ht5_exec(p, t, ip);
2909
2910         /* Thread. */
2911         thread_ip_inc(p);
2912 }
2913
2914 static inline void
2915 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2916 {
2917         struct thread *t = &p->threads[p->thread_id];
2918         struct instruction *ip = t->ip;
2919
2920         __instr_dma_ht6_exec(p, t, ip);
2921
2922         /* Thread. */
2923         thread_ip_inc(p);
2924 }
2925
2926 static inline void
2927 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2928 {
2929         struct thread *t = &p->threads[p->thread_id];
2930         struct instruction *ip = t->ip;
2931
2932         __instr_dma_ht7_exec(p, t, ip);
2933
2934         /* Thread. */
2935         thread_ip_inc(p);
2936 }
2937
2938 static inline void
2939 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2940 {
2941         struct thread *t = &p->threads[p->thread_id];
2942         struct instruction *ip = t->ip;
2943
2944         __instr_dma_ht8_exec(p, t, ip);
2945
2946         /* Thread. */
2947         thread_ip_inc(p);
2948 }
2949
2950 /*
2951  * alu.
2952  */
2953 static int
2954 instr_alu_add_translate(struct rte_swx_pipeline *p,
2955                         struct action *action,
2956                         char **tokens,
2957                         int n_tokens,
2958                         struct instruction *instr,
2959                         struct instruction_data *data __rte_unused)
2960 {
2961         char *dst = tokens[1], *src = tokens[2];
2962         struct field *fdst, *fsrc;
2963         uint64_t src_val;
2964         uint32_t dst_struct_id = 0, src_struct_id = 0;
2965
2966         CHECK(n_tokens == 3, EINVAL);
2967
2968         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2969         CHECK(fdst, EINVAL);
2970         CHECK(!fdst->var_size, EINVAL);
2971
2972         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2973         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2974         if (fsrc) {
2975                 CHECK(!fsrc->var_size, EINVAL);
2976
2977                 instr->type = INSTR_ALU_ADD;
2978                 if (dst[0] == 'h' && src[0] != 'h')
2979                         instr->type = INSTR_ALU_ADD_HM;
2980                 if (dst[0] != 'h' && src[0] == 'h')
2981                         instr->type = INSTR_ALU_ADD_MH;
2982                 if (dst[0] == 'h' && src[0] == 'h')
2983                         instr->type = INSTR_ALU_ADD_HH;
2984
2985                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2986                 instr->alu.dst.n_bits = fdst->n_bits;
2987                 instr->alu.dst.offset = fdst->offset / 8;
2988                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2989                 instr->alu.src.n_bits = fsrc->n_bits;
2990                 instr->alu.src.offset = fsrc->offset / 8;
2991                 return 0;
2992         }
2993
2994         /* ADD_MI, ADD_HI. */
2995         src_val = strtoull(src, &src, 0);
2996         CHECK(!src[0], EINVAL);
2997
2998         instr->type = INSTR_ALU_ADD_MI;
2999         if (dst[0] == 'h')
3000                 instr->type = INSTR_ALU_ADD_HI;
3001
3002         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3003         instr->alu.dst.n_bits = fdst->n_bits;
3004         instr->alu.dst.offset = fdst->offset / 8;
3005         instr->alu.src_val = src_val;
3006         return 0;
3007 }
3008
3009 static int
3010 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3011                         struct action *action,
3012                         char **tokens,
3013                         int n_tokens,
3014                         struct instruction *instr,
3015                         struct instruction_data *data __rte_unused)
3016 {
3017         char *dst = tokens[1], *src = tokens[2];
3018         struct field *fdst, *fsrc;
3019         uint64_t src_val;
3020         uint32_t dst_struct_id = 0, src_struct_id = 0;
3021
3022         CHECK(n_tokens == 3, EINVAL);
3023
3024         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3025         CHECK(fdst, EINVAL);
3026         CHECK(!fdst->var_size, EINVAL);
3027
3028         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3029         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3030         if (fsrc) {
3031                 CHECK(!fsrc->var_size, EINVAL);
3032
3033                 instr->type = INSTR_ALU_SUB;
3034                 if (dst[0] == 'h' && src[0] != 'h')
3035                         instr->type = INSTR_ALU_SUB_HM;
3036                 if (dst[0] != 'h' && src[0] == 'h')
3037                         instr->type = INSTR_ALU_SUB_MH;
3038                 if (dst[0] == 'h' && src[0] == 'h')
3039                         instr->type = INSTR_ALU_SUB_HH;
3040
3041                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3042                 instr->alu.dst.n_bits = fdst->n_bits;
3043                 instr->alu.dst.offset = fdst->offset / 8;
3044                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3045                 instr->alu.src.n_bits = fsrc->n_bits;
3046                 instr->alu.src.offset = fsrc->offset / 8;
3047                 return 0;
3048         }
3049
3050         /* SUB_MI, SUB_HI. */
3051         src_val = strtoull(src, &src, 0);
3052         CHECK(!src[0], EINVAL);
3053
3054         instr->type = INSTR_ALU_SUB_MI;
3055         if (dst[0] == 'h')
3056                 instr->type = INSTR_ALU_SUB_HI;
3057
3058         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3059         instr->alu.dst.n_bits = fdst->n_bits;
3060         instr->alu.dst.offset = fdst->offset / 8;
3061         instr->alu.src_val = src_val;
3062         return 0;
3063 }
3064
3065 static int
3066 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3067                           struct action *action __rte_unused,
3068                           char **tokens,
3069                           int n_tokens,
3070                           struct instruction *instr,
3071                           struct instruction_data *data __rte_unused)
3072 {
3073         char *dst = tokens[1], *src = tokens[2];
3074         struct header *hdst, *hsrc;
3075         struct field *fdst, *fsrc;
3076
3077         CHECK(n_tokens == 3, EINVAL);
3078
3079         fdst = header_field_parse(p, dst, &hdst);
3080         CHECK(fdst, EINVAL);
3081         CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3082
3083         /* CKADD_FIELD. */
3084         fsrc = header_field_parse(p, src, &hsrc);
3085         if (fsrc) {
3086                 CHECK(!fsrc->var_size, EINVAL);
3087
3088                 instr->type = INSTR_ALU_CKADD_FIELD;
3089                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3090                 instr->alu.dst.n_bits = fdst->n_bits;
3091                 instr->alu.dst.offset = fdst->offset / 8;
3092                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3093                 instr->alu.src.n_bits = fsrc->n_bits;
3094                 instr->alu.src.offset = fsrc->offset / 8;
3095                 return 0;
3096         }
3097
3098         /* CKADD_STRUCT, CKADD_STRUCT20. */
3099         hsrc = header_parse(p, src);
3100         CHECK(hsrc, EINVAL);
3101
3102         instr->type = INSTR_ALU_CKADD_STRUCT;
3103         if (!hsrc->st->var_size && ((hsrc->st->n_bits / 8) == 20))
3104                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3105
3106         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3107         instr->alu.dst.n_bits = fdst->n_bits;
3108         instr->alu.dst.offset = fdst->offset / 8;
3109         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3110         instr->alu.src.n_bits = (uint8_t)hsrc->id; /* The src header ID is stored here. */
3111         instr->alu.src.offset = 0; /* Unused. */
3112         return 0;
3113 }
3114
3115 static int
3116 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3117                           struct action *action __rte_unused,
3118                           char **tokens,
3119                           int n_tokens,
3120                           struct instruction *instr,
3121                           struct instruction_data *data __rte_unused)
3122 {
3123         char *dst = tokens[1], *src = tokens[2];
3124         struct header *hdst, *hsrc;
3125         struct field *fdst, *fsrc;
3126
3127         CHECK(n_tokens == 3, EINVAL);
3128
3129         fdst = header_field_parse(p, dst, &hdst);
3130         CHECK(fdst, EINVAL);
3131         CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3132
3133         fsrc = header_field_parse(p, src, &hsrc);
3134         CHECK(fsrc, EINVAL);
3135         CHECK(!fsrc->var_size, EINVAL);
3136
3137         instr->type = INSTR_ALU_CKSUB_FIELD;
3138         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3139         instr->alu.dst.n_bits = fdst->n_bits;
3140         instr->alu.dst.offset = fdst->offset / 8;
3141         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3142         instr->alu.src.n_bits = fsrc->n_bits;
3143         instr->alu.src.offset = fsrc->offset / 8;
3144         return 0;
3145 }
3146
3147 static int
3148 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3149                         struct action *action,
3150                         char **tokens,
3151                         int n_tokens,
3152                         struct instruction *instr,
3153                         struct instruction_data *data __rte_unused)
3154 {
3155         char *dst = tokens[1], *src = tokens[2];
3156         struct field *fdst, *fsrc;
3157         uint64_t src_val;
3158         uint32_t dst_struct_id = 0, src_struct_id = 0;
3159
3160         CHECK(n_tokens == 3, EINVAL);
3161
3162         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3163         CHECK(fdst, EINVAL);
3164         CHECK(!fdst->var_size, EINVAL);
3165
3166         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3167         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3168         if (fsrc) {
3169                 CHECK(!fsrc->var_size, EINVAL);
3170
3171                 instr->type = INSTR_ALU_SHL;
3172                 if (dst[0] == 'h' && src[0] != 'h')
3173                         instr->type = INSTR_ALU_SHL_HM;
3174                 if (dst[0] != 'h' && src[0] == 'h')
3175                         instr->type = INSTR_ALU_SHL_MH;
3176                 if (dst[0] == 'h' && src[0] == 'h')
3177                         instr->type = INSTR_ALU_SHL_HH;
3178
3179                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3180                 instr->alu.dst.n_bits = fdst->n_bits;
3181                 instr->alu.dst.offset = fdst->offset / 8;
3182                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3183                 instr->alu.src.n_bits = fsrc->n_bits;
3184                 instr->alu.src.offset = fsrc->offset / 8;
3185                 return 0;
3186         }
3187
3188         /* SHL_MI, SHL_HI. */
3189         src_val = strtoull(src, &src, 0);
3190         CHECK(!src[0], EINVAL);
3191
3192         instr->type = INSTR_ALU_SHL_MI;
3193         if (dst[0] == 'h')
3194                 instr->type = INSTR_ALU_SHL_HI;
3195
3196         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3197         instr->alu.dst.n_bits = fdst->n_bits;
3198         instr->alu.dst.offset = fdst->offset / 8;
3199         instr->alu.src_val = src_val;
3200         return 0;
3201 }
3202
3203 static int
3204 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3205                         struct action *action,
3206                         char **tokens,
3207                         int n_tokens,
3208                         struct instruction *instr,
3209                         struct instruction_data *data __rte_unused)
3210 {
3211         char *dst = tokens[1], *src = tokens[2];
3212         struct field *fdst, *fsrc;
3213         uint64_t src_val;
3214         uint32_t dst_struct_id = 0, src_struct_id = 0;
3215
3216         CHECK(n_tokens == 3, EINVAL);
3217
3218         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3219         CHECK(fdst, EINVAL);
3220         CHECK(!fdst->var_size, EINVAL);
3221
3222         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3223         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3224         if (fsrc) {
3225                 CHECK(!fsrc->var_size, EINVAL);
3226
3227                 instr->type = INSTR_ALU_SHR;
3228                 if (dst[0] == 'h' && src[0] != 'h')
3229                         instr->type = INSTR_ALU_SHR_HM;
3230                 if (dst[0] != 'h' && src[0] == 'h')
3231                         instr->type = INSTR_ALU_SHR_MH;
3232                 if (dst[0] == 'h' && src[0] == 'h')
3233                         instr->type = INSTR_ALU_SHR_HH;
3234
3235                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3236                 instr->alu.dst.n_bits = fdst->n_bits;
3237                 instr->alu.dst.offset = fdst->offset / 8;
3238                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3239                 instr->alu.src.n_bits = fsrc->n_bits;
3240                 instr->alu.src.offset = fsrc->offset / 8;
3241                 return 0;
3242         }
3243
3244         /* SHR_MI, SHR_HI. */
3245         src_val = strtoull(src, &src, 0);
3246         CHECK(!src[0], EINVAL);
3247
3248         instr->type = INSTR_ALU_SHR_MI;
3249         if (dst[0] == 'h')
3250                 instr->type = INSTR_ALU_SHR_HI;
3251
3252         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3253         instr->alu.dst.n_bits = fdst->n_bits;
3254         instr->alu.dst.offset = fdst->offset / 8;
3255         instr->alu.src_val = src_val;
3256         return 0;
3257 }
3258
3259 static int
3260 instr_alu_and_translate(struct rte_swx_pipeline *p,
3261                         struct action *action,
3262                         char **tokens,
3263                         int n_tokens,
3264                         struct instruction *instr,
3265                         struct instruction_data *data __rte_unused)
3266 {
3267         char *dst = tokens[1], *src = tokens[2];
3268         struct field *fdst, *fsrc;
3269         uint64_t src_val;
3270         uint32_t dst_struct_id = 0, src_struct_id = 0;
3271
3272         CHECK(n_tokens == 3, EINVAL);
3273
3274         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3275         CHECK(fdst, EINVAL);
3276         CHECK(!fdst->var_size, EINVAL);
3277
3278         /* AND, AND_MH, AND_HM, AND_HH. */
3279         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3280         if (fsrc) {
3281                 CHECK(!fsrc->var_size, EINVAL);
3282
3283                 instr->type = INSTR_ALU_AND;
3284                 if (dst[0] != 'h' && src[0] == 'h')
3285                         instr->type = INSTR_ALU_AND_MH;
3286                 if (dst[0] == 'h' && src[0] != 'h')
3287                         instr->type = INSTR_ALU_AND_HM;
3288                 if (dst[0] == 'h' && src[0] == 'h')
3289                         instr->type = INSTR_ALU_AND_HH;
3290
3291                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3292                 instr->alu.dst.n_bits = fdst->n_bits;
3293                 instr->alu.dst.offset = fdst->offset / 8;
3294                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3295                 instr->alu.src.n_bits = fsrc->n_bits;
3296                 instr->alu.src.offset = fsrc->offset / 8;
3297                 return 0;
3298         }
3299
3300         /* AND_I. */
3301         src_val = strtoull(src, &src, 0);
3302         CHECK(!src[0], EINVAL);
3303
3304         if (dst[0] == 'h')
3305                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3306
3307         instr->type = INSTR_ALU_AND_I;
3308         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3309         instr->alu.dst.n_bits = fdst->n_bits;
3310         instr->alu.dst.offset = fdst->offset / 8;
3311         instr->alu.src_val = src_val;
3312         return 0;
3313 }
3314
3315 static int
3316 instr_alu_or_translate(struct rte_swx_pipeline *p,
3317                        struct action *action,
3318                        char **tokens,
3319                        int n_tokens,
3320                        struct instruction *instr,
3321                        struct instruction_data *data __rte_unused)
3322 {
3323         char *dst = tokens[1], *src = tokens[2];
3324         struct field *fdst, *fsrc;
3325         uint64_t src_val;
3326         uint32_t dst_struct_id = 0, src_struct_id = 0;
3327
3328         CHECK(n_tokens == 3, EINVAL);
3329
3330         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3331         CHECK(fdst, EINVAL);
3332         CHECK(!fdst->var_size, EINVAL);
3333
3334         /* OR, OR_MH, OR_HM, OR_HH. */
3335         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3336         if (fsrc) {
3337                 CHECK(!fsrc->var_size, EINVAL);
3338
3339                 instr->type = INSTR_ALU_OR;
3340                 if (dst[0] != 'h' && src[0] == 'h')
3341                         instr->type = INSTR_ALU_OR_MH;
3342                 if (dst[0] == 'h' && src[0] != 'h')
3343                         instr->type = INSTR_ALU_OR_HM;
3344                 if (dst[0] == 'h' && src[0] == 'h')
3345                         instr->type = INSTR_ALU_OR_HH;
3346
3347                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3348                 instr->alu.dst.n_bits = fdst->n_bits;
3349                 instr->alu.dst.offset = fdst->offset / 8;
3350                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3351                 instr->alu.src.n_bits = fsrc->n_bits;
3352                 instr->alu.src.offset = fsrc->offset / 8;
3353                 return 0;
3354         }
3355
3356         /* OR_I. */
3357         src_val = strtoull(src, &src, 0);
3358         CHECK(!src[0], EINVAL);
3359
3360         if (dst[0] == 'h')
3361                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3362
3363         instr->type = INSTR_ALU_OR_I;
3364         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3365         instr->alu.dst.n_bits = fdst->n_bits;
3366         instr->alu.dst.offset = fdst->offset / 8;
3367         instr->alu.src_val = src_val;
3368         return 0;
3369 }
3370
3371 static int
3372 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3373                         struct action *action,
3374                         char **tokens,
3375                         int n_tokens,
3376                         struct instruction *instr,
3377                         struct instruction_data *data __rte_unused)
3378 {
3379         char *dst = tokens[1], *src = tokens[2];
3380         struct field *fdst, *fsrc;
3381         uint64_t src_val;
3382         uint32_t dst_struct_id = 0, src_struct_id = 0;
3383
3384         CHECK(n_tokens == 3, EINVAL);
3385
3386         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3387         CHECK(fdst, EINVAL);
3388         CHECK(!fdst->var_size, EINVAL);
3389
3390         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3391         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3392         if (fsrc) {
3393                 CHECK(!fsrc->var_size, EINVAL);
3394
3395                 instr->type = INSTR_ALU_XOR;
3396                 if (dst[0] != 'h' && src[0] == 'h')
3397                         instr->type = INSTR_ALU_XOR_MH;
3398                 if (dst[0] == 'h' && src[0] != 'h')
3399                         instr->type = INSTR_ALU_XOR_HM;
3400                 if (dst[0] == 'h' && src[0] == 'h')
3401                         instr->type = INSTR_ALU_XOR_HH;
3402
3403                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3404                 instr->alu.dst.n_bits = fdst->n_bits;
3405                 instr->alu.dst.offset = fdst->offset / 8;
3406                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3407                 instr->alu.src.n_bits = fsrc->n_bits;
3408                 instr->alu.src.offset = fsrc->offset / 8;
3409                 return 0;
3410         }
3411
3412         /* XOR_I. */
3413         src_val = strtoull(src, &src, 0);
3414         CHECK(!src[0], EINVAL);
3415
3416         if (dst[0] == 'h')
3417                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3418
3419         instr->type = INSTR_ALU_XOR_I;
3420         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3421         instr->alu.dst.n_bits = fdst->n_bits;
3422         instr->alu.dst.offset = fdst->offset / 8;
3423         instr->alu.src_val = src_val;
3424         return 0;
3425 }
3426
3427 static inline void
3428 instr_alu_add_exec(struct rte_swx_pipeline *p)
3429 {
3430         struct thread *t = &p->threads[p->thread_id];
3431         struct instruction *ip = t->ip;
3432
3433         /* Structs */
3434         __instr_alu_add_exec(p, t, ip);
3435
3436         /* Thread. */
3437         thread_ip_inc(p);
3438 }
3439
3440 static inline void
3441 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3442 {
3443         struct thread *t = &p->threads[p->thread_id];
3444         struct instruction *ip = t->ip;
3445
3446         /* Structs. */
3447         __instr_alu_add_mh_exec(p, t, ip);
3448
3449         /* Thread. */
3450         thread_ip_inc(p);
3451 }
3452
3453 static inline void
3454 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3455 {
3456         struct thread *t = &p->threads[p->thread_id];
3457         struct instruction *ip = t->ip;
3458
3459         /* Structs. */
3460         __instr_alu_add_hm_exec(p, t, ip);
3461
3462         /* Thread. */
3463         thread_ip_inc(p);
3464 }
3465
3466 static inline void
3467 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3468 {
3469         struct thread *t = &p->threads[p->thread_id];
3470         struct instruction *ip = t->ip;
3471
3472         /* Structs. */
3473         __instr_alu_add_hh_exec(p, t, ip);
3474
3475         /* Thread. */
3476         thread_ip_inc(p);
3477 }
3478
3479 static inline void
3480 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3481 {
3482         struct thread *t = &p->threads[p->thread_id];
3483         struct instruction *ip = t->ip;
3484
3485         /* Structs. */
3486         __instr_alu_add_mi_exec(p, t, ip);
3487
3488         /* Thread. */
3489         thread_ip_inc(p);
3490 }
3491
3492 static inline void
3493 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3494 {
3495         struct thread *t = &p->threads[p->thread_id];
3496         struct instruction *ip = t->ip;
3497
3498         /* Structs. */
3499         __instr_alu_add_hi_exec(p, t, ip);
3500
3501         /* Thread. */
3502         thread_ip_inc(p);
3503 }
3504
3505 static inline void
3506 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3507 {
3508         struct thread *t = &p->threads[p->thread_id];
3509         struct instruction *ip = t->ip;
3510
3511         /* Structs. */
3512         __instr_alu_sub_exec(p, t, ip);
3513
3514         /* Thread. */
3515         thread_ip_inc(p);
3516 }
3517
3518 static inline void
3519 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3520 {
3521         struct thread *t = &p->threads[p->thread_id];
3522         struct instruction *ip = t->ip;
3523
3524         /* Structs. */
3525         __instr_alu_sub_mh_exec(p, t, ip);
3526
3527         /* Thread. */
3528         thread_ip_inc(p);
3529 }
3530
3531 static inline void
3532 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3533 {
3534         struct thread *t = &p->threads[p->thread_id];
3535         struct instruction *ip = t->ip;
3536
3537         /* Structs. */
3538         __instr_alu_sub_hm_exec(p, t, ip);
3539
3540         /* Thread. */
3541         thread_ip_inc(p);
3542 }
3543
3544 static inline void
3545 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3546 {
3547         struct thread *t = &p->threads[p->thread_id];
3548         struct instruction *ip = t->ip;
3549
3550         /* Structs. */
3551         __instr_alu_sub_hh_exec(p, t, ip);
3552
3553         /* Thread. */
3554         thread_ip_inc(p);
3555 }
3556
3557 static inline void
3558 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3559 {
3560         struct thread *t = &p->threads[p->thread_id];
3561         struct instruction *ip = t->ip;
3562
3563         /* Structs. */
3564         __instr_alu_sub_mi_exec(p, t, ip);
3565
3566         /* Thread. */
3567         thread_ip_inc(p);
3568 }
3569
3570 static inline void
3571 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3572 {
3573         struct thread *t = &p->threads[p->thread_id];
3574         struct instruction *ip = t->ip;
3575
3576         /* Structs. */
3577         __instr_alu_sub_hi_exec(p, t, ip);
3578
3579         /* Thread. */
3580         thread_ip_inc(p);
3581 }
3582
3583 static inline void
3584 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3585 {
3586         struct thread *t = &p->threads[p->thread_id];
3587         struct instruction *ip = t->ip;
3588
3589         /* Structs. */
3590         __instr_alu_shl_exec(p, t, ip);
3591
3592         /* Thread. */
3593         thread_ip_inc(p);
3594 }
3595
3596 static inline void
3597 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3598 {
3599         struct thread *t = &p->threads[p->thread_id];
3600         struct instruction *ip = t->ip;
3601
3602         /* Structs. */
3603         __instr_alu_shl_mh_exec(p, t, ip);
3604
3605         /* Thread. */
3606         thread_ip_inc(p);
3607 }
3608
3609 static inline void
3610 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3611 {
3612         struct thread *t = &p->threads[p->thread_id];
3613         struct instruction *ip = t->ip;
3614
3615         /* Structs. */
3616         __instr_alu_shl_hm_exec(p, t, ip);
3617
3618         /* Thread. */
3619         thread_ip_inc(p);
3620 }
3621
3622 static inline void
3623 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3624 {
3625         struct thread *t = &p->threads[p->thread_id];
3626         struct instruction *ip = t->ip;
3627
3628         /* Structs. */
3629         __instr_alu_shl_hh_exec(p, t, ip);
3630
3631         /* Thread. */
3632         thread_ip_inc(p);
3633 }
3634
3635 static inline void
3636 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3637 {
3638         struct thread *t = &p->threads[p->thread_id];
3639         struct instruction *ip = t->ip;
3640
3641         /* Structs. */
3642         __instr_alu_shl_mi_exec(p, t, ip);
3643
3644         /* Thread. */
3645         thread_ip_inc(p);
3646 }
3647
3648 static inline void
3649 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3650 {
3651         struct thread *t = &p->threads[p->thread_id];
3652         struct instruction *ip = t->ip;
3653
3654         /* Structs. */
3655         __instr_alu_shl_hi_exec(p, t, ip);
3656
3657         /* Thread. */
3658         thread_ip_inc(p);
3659 }
3660
3661 static inline void
3662 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3663 {
3664         struct thread *t = &p->threads[p->thread_id];
3665         struct instruction *ip = t->ip;
3666
3667         /* Structs. */
3668         __instr_alu_shr_exec(p, t, ip);
3669
3670         /* Thread. */
3671         thread_ip_inc(p);
3672 }
3673
3674 static inline void
3675 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3676 {
3677         struct thread *t = &p->threads[p->thread_id];
3678         struct instruction *ip = t->ip;
3679
3680         /* Structs. */
3681         __instr_alu_shr_mh_exec(p, t, ip);
3682
3683         /* Thread. */
3684         thread_ip_inc(p);
3685 }
3686
3687 static inline void
3688 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3689 {
3690         struct thread *t = &p->threads[p->thread_id];
3691         struct instruction *ip = t->ip;
3692
3693         /* Structs. */
3694         __instr_alu_shr_hm_exec(p, t, ip);
3695
3696         /* Thread. */
3697         thread_ip_inc(p);
3698 }
3699
3700 static inline void
3701 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3702 {
3703         struct thread *t = &p->threads[p->thread_id];
3704         struct instruction *ip = t->ip;
3705
3706         /* Structs. */
3707         __instr_alu_shr_hh_exec(p, t, ip);
3708
3709         /* Thread. */
3710         thread_ip_inc(p);
3711 }
3712
3713 static inline void
3714 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3715 {
3716         struct thread *t = &p->threads[p->thread_id];
3717         struct instruction *ip = t->ip;
3718
3719         /* Structs. */
3720         __instr_alu_shr_mi_exec(p, t, ip);
3721
3722         /* Thread. */
3723         thread_ip_inc(p);
3724 }
3725
3726 static inline void
3727 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3728 {
3729         struct thread *t = &p->threads[p->thread_id];
3730         struct instruction *ip = t->ip;
3731
3732         /* Structs. */
3733         __instr_alu_shr_hi_exec(p, t, ip);
3734
3735         /* Thread. */
3736         thread_ip_inc(p);
3737 }
3738
3739 static inline void
3740 instr_alu_and_exec(struct rte_swx_pipeline *p)
3741 {
3742         struct thread *t = &p->threads[p->thread_id];
3743         struct instruction *ip = t->ip;
3744
3745         /* Structs. */
3746         __instr_alu_and_exec(p, t, ip);
3747
3748         /* Thread. */
3749         thread_ip_inc(p);
3750 }
3751
3752 static inline void
3753 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3754 {
3755         struct thread *t = &p->threads[p->thread_id];
3756         struct instruction *ip = t->ip;
3757
3758         /* Structs. */
3759         __instr_alu_and_mh_exec(p, t, ip);
3760
3761         /* Thread. */
3762         thread_ip_inc(p);
3763 }
3764
3765 static inline void
3766 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3767 {
3768         struct thread *t = &p->threads[p->thread_id];
3769         struct instruction *ip = t->ip;
3770
3771         /* Structs. */
3772         __instr_alu_and_hm_exec(p, t, ip);
3773
3774         /* Thread. */
3775         thread_ip_inc(p);
3776 }
3777
3778 static inline void
3779 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3780 {
3781         struct thread *t = &p->threads[p->thread_id];
3782         struct instruction *ip = t->ip;
3783
3784         /* Structs. */
3785         __instr_alu_and_hh_exec(p, t, ip);
3786
3787         /* Thread. */
3788         thread_ip_inc(p);
3789 }
3790
3791 static inline void
3792 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3793 {
3794         struct thread *t = &p->threads[p->thread_id];
3795         struct instruction *ip = t->ip;
3796
3797         /* Structs. */
3798         __instr_alu_and_i_exec(p, t, ip);
3799
3800         /* Thread. */
3801         thread_ip_inc(p);
3802 }
3803
3804 static inline void
3805 instr_alu_or_exec(struct rte_swx_pipeline *p)
3806 {
3807         struct thread *t = &p->threads[p->thread_id];
3808         struct instruction *ip = t->ip;
3809
3810         /* Structs. */
3811         __instr_alu_or_exec(p, t, ip);
3812
3813         /* Thread. */
3814         thread_ip_inc(p);
3815 }
3816
3817 static inline void
3818 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3819 {
3820         struct thread *t = &p->threads[p->thread_id];
3821         struct instruction *ip = t->ip;
3822
3823         /* Structs. */
3824         __instr_alu_or_mh_exec(p, t, ip);
3825
3826         /* Thread. */
3827         thread_ip_inc(p);
3828 }
3829
3830 static inline void
3831 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3832 {
3833         struct thread *t = &p->threads[p->thread_id];
3834         struct instruction *ip = t->ip;
3835
3836         /* Structs. */
3837         __instr_alu_or_hm_exec(p, t, ip);
3838
3839         /* Thread. */
3840         thread_ip_inc(p);
3841 }
3842
3843 static inline void
3844 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3845 {
3846         struct thread *t = &p->threads[p->thread_id];
3847         struct instruction *ip = t->ip;
3848
3849         /* Structs. */
3850         __instr_alu_or_hh_exec(p, t, ip);
3851
3852         /* Thread. */
3853         thread_ip_inc(p);
3854 }
3855
3856 static inline void
3857 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3858 {
3859         struct thread *t = &p->threads[p->thread_id];
3860         struct instruction *ip = t->ip;
3861
3862         /* Structs. */
3863         __instr_alu_or_i_exec(p, t, ip);
3864
3865         /* Thread. */
3866         thread_ip_inc(p);
3867 }
3868
3869 static inline void
3870 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3871 {
3872         struct thread *t = &p->threads[p->thread_id];
3873         struct instruction *ip = t->ip;
3874
3875         /* Structs. */
3876         __instr_alu_xor_exec(p, t, ip);
3877
3878         /* Thread. */
3879         thread_ip_inc(p);
3880 }
3881
3882 static inline void
3883 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3884 {
3885         struct thread *t = &p->threads[p->thread_id];
3886         struct instruction *ip = t->ip;
3887
3888         /* Structs. */
3889         __instr_alu_xor_mh_exec(p, t, ip);
3890
3891         /* Thread. */
3892         thread_ip_inc(p);
3893 }
3894
3895 static inline void
3896 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3897 {
3898         struct thread *t = &p->threads[p->thread_id];
3899         struct instruction *ip = t->ip;
3900
3901         /* Structs. */
3902         __instr_alu_xor_hm_exec(p, t, ip);
3903
3904         /* Thread. */
3905         thread_ip_inc(p);
3906 }
3907
3908 static inline void
3909 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3910 {
3911         struct thread *t = &p->threads[p->thread_id];
3912         struct instruction *ip = t->ip;
3913
3914         /* Structs. */
3915         __instr_alu_xor_hh_exec(p, t, ip);
3916
3917         /* Thread. */
3918         thread_ip_inc(p);
3919 }
3920
3921 static inline void
3922 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3923 {
3924         struct thread *t = &p->threads[p->thread_id];
3925         struct instruction *ip = t->ip;
3926
3927         /* Structs. */
3928         __instr_alu_xor_i_exec(p, t, ip);
3929
3930         /* Thread. */
3931         thread_ip_inc(p);
3932 }
3933
3934 static inline void
3935 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3936 {
3937         struct thread *t = &p->threads[p->thread_id];
3938         struct instruction *ip = t->ip;
3939
3940         /* Structs. */
3941         __instr_alu_ckadd_field_exec(p, t, ip);
3942
3943         /* Thread. */
3944         thread_ip_inc(p);
3945 }
3946
3947 static inline void
3948 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3949 {
3950         struct thread *t = &p->threads[p->thread_id];
3951         struct instruction *ip = t->ip;
3952
3953         /* Structs. */
3954         __instr_alu_cksub_field_exec(p, t, ip);
3955
3956         /* Thread. */
3957         thread_ip_inc(p);
3958 }
3959
3960 static inline void
3961 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3962 {
3963         struct thread *t = &p->threads[p->thread_id];
3964         struct instruction *ip = t->ip;
3965
3966         /* Structs. */
3967         __instr_alu_ckadd_struct20_exec(p, t, ip);
3968
3969         /* Thread. */
3970         thread_ip_inc(p);
3971 }
3972
3973 static inline void
3974 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3975 {
3976         struct thread *t = &p->threads[p->thread_id];
3977         struct instruction *ip = t->ip;
3978
3979         /* Structs. */
3980         __instr_alu_ckadd_struct_exec(p, t, ip);
3981
3982         /* Thread. */
3983         thread_ip_inc(p);
3984 }
3985
3986 /*
3987  * Register array.
3988  */
3989 static struct regarray *
3990 regarray_find(struct rte_swx_pipeline *p, const char *name);
3991
3992 static int
3993 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3994                       struct action *action,
3995                       char **tokens,
3996                       int n_tokens,
3997                       struct instruction *instr,
3998                       struct instruction_data *data __rte_unused)
3999 {
4000         char *regarray = tokens[1], *idx = tokens[2];
4001         struct regarray *r;
4002         struct field *fidx;
4003         uint32_t idx_struct_id, idx_val;
4004
4005         CHECK(n_tokens == 3, EINVAL);
4006
4007         r = regarray_find(p, regarray);
4008         CHECK(r, EINVAL);
4009
4010         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4011         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4012         if (fidx) {
4013                 CHECK(!fidx->var_size, EINVAL);
4014
4015                 instr->type = INSTR_REGPREFETCH_RM;
4016                 if (idx[0] == 'h')
4017                         instr->type = INSTR_REGPREFETCH_RH;
4018
4019                 instr->regarray.regarray_id = r->id;
4020                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4021                 instr->regarray.idx.n_bits = fidx->n_bits;
4022                 instr->regarray.idx.offset = fidx->offset / 8;
4023                 instr->regarray.dstsrc_val = 0; /* Unused. */
4024                 return 0;
4025         }
4026
4027         /* REGPREFETCH_RI. */
4028         idx_val = strtoul(idx, &idx, 0);
4029         CHECK(!idx[0], EINVAL);
4030
4031         instr->type = INSTR_REGPREFETCH_RI;
4032         instr->regarray.regarray_id = r->id;
4033         instr->regarray.idx_val = idx_val;
4034         instr->regarray.dstsrc_val = 0; /* Unused. */
4035         return 0;
4036 }
4037
4038 static int
4039 instr_regrd_translate(struct rte_swx_pipeline *p,
4040                       struct action *action,
4041                       char **tokens,
4042                       int n_tokens,
4043                       struct instruction *instr,
4044                       struct instruction_data *data __rte_unused)
4045 {
4046         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4047         struct regarray *r;
4048         struct field *fdst, *fidx;
4049         uint32_t dst_struct_id, idx_struct_id, idx_val;
4050
4051         CHECK(n_tokens == 4, EINVAL);
4052
4053         r = regarray_find(p, regarray);
4054         CHECK(r, EINVAL);
4055
4056         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4057         CHECK(fdst, EINVAL);
4058         CHECK(!fdst->var_size, EINVAL);
4059
4060         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4061         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4062         if (fidx) {
4063                 CHECK(!fidx->var_size, EINVAL);
4064
4065                 instr->type = INSTR_REGRD_MRM;
4066                 if (dst[0] == 'h' && idx[0] != 'h')
4067                         instr->type = INSTR_REGRD_HRM;
4068                 if (dst[0] != 'h' && idx[0] == 'h')
4069                         instr->type = INSTR_REGRD_MRH;
4070                 if (dst[0] == 'h' && idx[0] == 'h')
4071                         instr->type = INSTR_REGRD_HRH;
4072
4073                 instr->regarray.regarray_id = r->id;
4074                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4075                 instr->regarray.idx.n_bits = fidx->n_bits;
4076                 instr->regarray.idx.offset = fidx->offset / 8;
4077                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4078                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4079                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4080                 return 0;
4081         }
4082
4083         /* REGRD_MRI, REGRD_HRI. */
4084         idx_val = strtoul(idx, &idx, 0);
4085         CHECK(!idx[0], EINVAL);
4086
4087         instr->type = INSTR_REGRD_MRI;
4088         if (dst[0] == 'h')
4089                 instr->type = INSTR_REGRD_HRI;
4090
4091         instr->regarray.regarray_id = r->id;
4092         instr->regarray.idx_val = idx_val;
4093         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4094         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4095         instr->regarray.dstsrc.offset = fdst->offset / 8;
4096         return 0;
4097 }
4098
4099 static int
4100 instr_regwr_translate(struct rte_swx_pipeline *p,
4101                       struct action *action,
4102                       char **tokens,
4103                       int n_tokens,
4104                       struct instruction *instr,
4105                       struct instruction_data *data __rte_unused)
4106 {
4107         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4108         struct regarray *r;
4109         struct field *fidx, *fsrc;
4110         uint64_t src_val;
4111         uint32_t idx_struct_id, idx_val, src_struct_id;
4112
4113         CHECK(n_tokens == 4, EINVAL);
4114
4115         r = regarray_find(p, regarray);
4116         CHECK(r, EINVAL);
4117
4118         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4119         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4120         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4121         if (fidx && fsrc) {
4122                 CHECK(!fidx->var_size, EINVAL);
4123                 CHECK(!fsrc->var_size, EINVAL);
4124
4125                 instr->type = INSTR_REGWR_RMM;
4126                 if (idx[0] == 'h' && src[0] != 'h')
4127                         instr->type = INSTR_REGWR_RHM;
4128                 if (idx[0] != 'h' && src[0] == 'h')
4129                         instr->type = INSTR_REGWR_RMH;
4130                 if (idx[0] == 'h' && src[0] == 'h')
4131                         instr->type = INSTR_REGWR_RHH;
4132
4133                 instr->regarray.regarray_id = r->id;
4134                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4135                 instr->regarray.idx.n_bits = fidx->n_bits;
4136                 instr->regarray.idx.offset = fidx->offset / 8;
4137                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4138                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4139                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4140                 return 0;
4141         }
4142
4143         /* REGWR_RHI, REGWR_RMI. */
4144         if (fidx && !fsrc) {
4145                 CHECK(!fidx->var_size, EINVAL);
4146
4147                 src_val = strtoull(src, &src, 0);
4148                 CHECK(!src[0], EINVAL);
4149
4150                 instr->type = INSTR_REGWR_RMI;
4151                 if (idx[0] == 'h')
4152                         instr->type = INSTR_REGWR_RHI;
4153
4154                 instr->regarray.regarray_id = r->id;
4155                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4156                 instr->regarray.idx.n_bits = fidx->n_bits;
4157                 instr->regarray.idx.offset = fidx->offset / 8;
4158                 instr->regarray.dstsrc_val = src_val;
4159                 return 0;
4160         }
4161
4162         /* REGWR_RIH, REGWR_RIM. */
4163         if (!fidx && fsrc) {
4164                 idx_val = strtoul(idx, &idx, 0);
4165                 CHECK(!idx[0], EINVAL);
4166
4167                 CHECK(!fsrc->var_size, EINVAL);
4168
4169                 instr->type = INSTR_REGWR_RIM;
4170                 if (src[0] == 'h')
4171                         instr->type = INSTR_REGWR_RIH;
4172
4173                 instr->regarray.regarray_id = r->id;
4174                 instr->regarray.idx_val = idx_val;
4175                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4176                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4177                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4178                 return 0;
4179         }
4180
4181         /* REGWR_RII. */
4182         src_val = strtoull(src, &src, 0);
4183         CHECK(!src[0], EINVAL);
4184
4185         idx_val = strtoul(idx, &idx, 0);
4186         CHECK(!idx[0], EINVAL);
4187
4188         instr->type = INSTR_REGWR_RII;
4189         instr->regarray.idx_val = idx_val;
4190         instr->regarray.dstsrc_val = src_val;
4191
4192         return 0;
4193 }
4194
4195 static int
4196 instr_regadd_translate(struct rte_swx_pipeline *p,
4197                        struct action *action,
4198                        char **tokens,
4199                        int n_tokens,
4200                        struct instruction *instr,
4201                        struct instruction_data *data __rte_unused)
4202 {
4203         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4204         struct regarray *r;
4205         struct field *fidx, *fsrc;
4206         uint64_t src_val;
4207         uint32_t idx_struct_id, idx_val, src_struct_id;
4208
4209         CHECK(n_tokens == 4, EINVAL);
4210
4211         r = regarray_find(p, regarray);
4212         CHECK(r, EINVAL);
4213
4214         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4215         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4216         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4217         if (fidx && fsrc) {
4218                 CHECK(!fidx->var_size, EINVAL);
4219                 CHECK(!fsrc->var_size, EINVAL);
4220
4221                 instr->type = INSTR_REGADD_RMM;
4222                 if (idx[0] == 'h' && src[0] != 'h')
4223                         instr->type = INSTR_REGADD_RHM;
4224                 if (idx[0] != 'h' && src[0] == 'h')
4225                         instr->type = INSTR_REGADD_RMH;
4226                 if (idx[0] == 'h' && src[0] == 'h')
4227                         instr->type = INSTR_REGADD_RHH;
4228
4229                 instr->regarray.regarray_id = r->id;
4230                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4231                 instr->regarray.idx.n_bits = fidx->n_bits;
4232                 instr->regarray.idx.offset = fidx->offset / 8;
4233                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4234                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4235                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4236                 return 0;
4237         }
4238
4239         /* REGADD_RHI, REGADD_RMI. */
4240         if (fidx && !fsrc) {
4241                 CHECK(!fidx->var_size, EINVAL);
4242
4243                 src_val = strtoull(src, &src, 0);
4244                 CHECK(!src[0], EINVAL);
4245
4246                 instr->type = INSTR_REGADD_RMI;
4247                 if (idx[0] == 'h')
4248                         instr->type = INSTR_REGADD_RHI;
4249
4250                 instr->regarray.regarray_id = r->id;
4251                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4252                 instr->regarray.idx.n_bits = fidx->n_bits;
4253                 instr->regarray.idx.offset = fidx->offset / 8;
4254                 instr->regarray.dstsrc_val = src_val;
4255                 return 0;
4256         }
4257
4258         /* REGADD_RIH, REGADD_RIM. */
4259         if (!fidx && fsrc) {
4260                 idx_val = strtoul(idx, &idx, 0);
4261                 CHECK(!idx[0], EINVAL);
4262
4263                 CHECK(!fsrc->var_size, EINVAL);
4264
4265                 instr->type = INSTR_REGADD_RIM;
4266                 if (src[0] == 'h')
4267                         instr->type = INSTR_REGADD_RIH;
4268
4269                 instr->regarray.regarray_id = r->id;
4270                 instr->regarray.idx_val = idx_val;
4271                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4272                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4273                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4274                 return 0;
4275         }
4276
4277         /* REGADD_RII. */
4278         src_val = strtoull(src, &src, 0);
4279         CHECK(!src[0], EINVAL);
4280
4281         idx_val = strtoul(idx, &idx, 0);
4282         CHECK(!idx[0], EINVAL);
4283
4284         instr->type = INSTR_REGADD_RII;
4285         instr->regarray.idx_val = idx_val;
4286         instr->regarray.dstsrc_val = src_val;
4287         return 0;
4288 }
4289
4290 static inline void
4291 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4292 {
4293         struct thread *t = &p->threads[p->thread_id];
4294         struct instruction *ip = t->ip;
4295
4296         /* Structs. */
4297         __instr_regprefetch_rh_exec(p, t, ip);
4298
4299         /* Thread. */
4300         thread_ip_inc(p);
4301 }
4302
4303 static inline void
4304 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4305 {
4306         struct thread *t = &p->threads[p->thread_id];
4307         struct instruction *ip = t->ip;
4308
4309         /* Structs. */
4310         __instr_regprefetch_rm_exec(p, t, ip);
4311
4312         /* Thread. */
4313         thread_ip_inc(p);
4314 }
4315
4316 static inline void
4317 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4318 {
4319         struct thread *t = &p->threads[p->thread_id];
4320         struct instruction *ip = t->ip;
4321
4322         /* Structs. */
4323         __instr_regprefetch_ri_exec(p, t, ip);
4324
4325         /* Thread. */
4326         thread_ip_inc(p);
4327 }
4328
4329 static inline void
4330 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4331 {
4332         struct thread *t = &p->threads[p->thread_id];
4333         struct instruction *ip = t->ip;
4334
4335         /* Structs. */
4336         __instr_regrd_hrh_exec(p, t, ip);
4337
4338         /* Thread. */
4339         thread_ip_inc(p);
4340 }
4341
4342 static inline void
4343 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4344 {
4345         struct thread *t = &p->threads[p->thread_id];
4346         struct instruction *ip = t->ip;
4347
4348         /* Structs. */
4349         __instr_regrd_hrm_exec(p, t, ip);
4350
4351         /* Thread. */
4352         thread_ip_inc(p);
4353 }
4354
4355 static inline void
4356 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4357 {
4358         struct thread *t = &p->threads[p->thread_id];
4359         struct instruction *ip = t->ip;
4360
4361         /* Structs. */
4362         __instr_regrd_mrh_exec(p, t, ip);
4363
4364         /* Thread. */
4365         thread_ip_inc(p);
4366 }
4367
4368 static inline void
4369 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4370 {
4371         struct thread *t = &p->threads[p->thread_id];
4372         struct instruction *ip = t->ip;
4373
4374         /* Structs. */
4375         __instr_regrd_mrm_exec(p, t, ip);
4376
4377         /* Thread. */
4378         thread_ip_inc(p);
4379 }
4380
4381 static inline void
4382 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4383 {
4384         struct thread *t = &p->threads[p->thread_id];
4385         struct instruction *ip = t->ip;
4386
4387         /* Structs. */
4388         __instr_regrd_hri_exec(p, t, ip);
4389
4390         /* Thread. */
4391         thread_ip_inc(p);
4392 }
4393
4394 static inline void
4395 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4396 {
4397         struct thread *t = &p->threads[p->thread_id];
4398         struct instruction *ip = t->ip;
4399
4400         /* Structs. */
4401         __instr_regrd_mri_exec(p, t, ip);
4402
4403         /* Thread. */
4404         thread_ip_inc(p);
4405 }
4406
4407 static inline void
4408 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4409 {
4410         struct thread *t = &p->threads[p->thread_id];
4411         struct instruction *ip = t->ip;
4412
4413         /* Structs. */
4414         __instr_regwr_rhh_exec(p, t, ip);
4415
4416         /* Thread. */
4417         thread_ip_inc(p);
4418 }
4419
4420 static inline void
4421 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4422 {
4423         struct thread *t = &p->threads[p->thread_id];
4424         struct instruction *ip = t->ip;
4425
4426         /* Structs. */
4427         __instr_regwr_rhm_exec(p, t, ip);
4428
4429         /* Thread. */
4430         thread_ip_inc(p);
4431 }
4432
4433 static inline void
4434 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4435 {
4436         struct thread *t = &p->threads[p->thread_id];
4437         struct instruction *ip = t->ip;
4438
4439         /* Structs. */
4440         __instr_regwr_rmh_exec(p, t, ip);
4441
4442         /* Thread. */
4443         thread_ip_inc(p);
4444 }
4445
4446 static inline void
4447 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4448 {
4449         struct thread *t = &p->threads[p->thread_id];
4450         struct instruction *ip = t->ip;
4451
4452         /* Structs. */
4453         __instr_regwr_rmm_exec(p, t, ip);
4454
4455         /* Thread. */
4456         thread_ip_inc(p);
4457 }
4458
4459 static inline void
4460 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4461 {
4462         struct thread *t = &p->threads[p->thread_id];
4463         struct instruction *ip = t->ip;
4464
4465         /* Structs. */
4466         __instr_regwr_rhi_exec(p, t, ip);
4467
4468         /* Thread. */
4469         thread_ip_inc(p);
4470 }
4471
4472 static inline void
4473 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4474 {
4475         struct thread *t = &p->threads[p->thread_id];
4476         struct instruction *ip = t->ip;
4477
4478         /* Structs. */
4479         __instr_regwr_rmi_exec(p, t, ip);
4480
4481         /* Thread. */
4482         thread_ip_inc(p);
4483 }
4484
4485 static inline void
4486 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4487 {
4488         struct thread *t = &p->threads[p->thread_id];
4489         struct instruction *ip = t->ip;
4490
4491         /* Structs. */
4492         __instr_regwr_rih_exec(p, t, ip);
4493
4494         /* Thread. */
4495         thread_ip_inc(p);
4496 }
4497
4498 static inline void
4499 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4500 {
4501         struct thread *t = &p->threads[p->thread_id];
4502         struct instruction *ip = t->ip;
4503
4504         /* Structs. */
4505         __instr_regwr_rim_exec(p, t, ip);
4506
4507         /* Thread. */
4508         thread_ip_inc(p);
4509 }
4510
4511 static inline void
4512 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4513 {
4514         struct thread *t = &p->threads[p->thread_id];
4515         struct instruction *ip = t->ip;
4516
4517         /* Structs. */
4518         __instr_regwr_rii_exec(p, t, ip);
4519
4520         /* Thread. */
4521         thread_ip_inc(p);
4522 }
4523
4524 static inline void
4525 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4526 {
4527         struct thread *t = &p->threads[p->thread_id];
4528         struct instruction *ip = t->ip;
4529
4530         /* Structs. */
4531         __instr_regadd_rhh_exec(p, t, ip);
4532
4533         /* Thread. */
4534         thread_ip_inc(p);
4535 }
4536
4537 static inline void
4538 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4539 {
4540         struct thread *t = &p->threads[p->thread_id];
4541         struct instruction *ip = t->ip;
4542
4543         /* Structs. */
4544         __instr_regadd_rhm_exec(p, t, ip);
4545
4546         /* Thread. */
4547         thread_ip_inc(p);
4548 }
4549
4550 static inline void
4551 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4552 {
4553         struct thread *t = &p->threads[p->thread_id];
4554         struct instruction *ip = t->ip;
4555
4556         /* Structs. */
4557         __instr_regadd_rmh_exec(p, t, ip);
4558
4559         /* Thread. */
4560         thread_ip_inc(p);
4561 }
4562
4563 static inline void
4564 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4565 {
4566         struct thread *t = &p->threads[p->thread_id];
4567         struct instruction *ip = t->ip;
4568
4569         /* Structs. */
4570         __instr_regadd_rmm_exec(p, t, ip);
4571
4572         /* Thread. */
4573         thread_ip_inc(p);
4574 }
4575
4576 static inline void
4577 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4578 {
4579         struct thread *t = &p->threads[p->thread_id];
4580         struct instruction *ip = t->ip;
4581
4582         /* Structs. */
4583         __instr_regadd_rhi_exec(p, t, ip);
4584
4585         /* Thread. */
4586         thread_ip_inc(p);
4587 }
4588
4589 static inline void
4590 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4591 {
4592         struct thread *t = &p->threads[p->thread_id];
4593         struct instruction *ip = t->ip;
4594
4595         /* Structs. */
4596         __instr_regadd_rmi_exec(p, t, ip);
4597
4598         /* Thread. */
4599         thread_ip_inc(p);
4600 }
4601
4602 static inline void
4603 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4604 {
4605         struct thread *t = &p->threads[p->thread_id];
4606         struct instruction *ip = t->ip;
4607
4608         /* Structs. */
4609         __instr_regadd_rih_exec(p, t, ip);
4610
4611         /* Thread. */
4612         thread_ip_inc(p);
4613 }
4614
4615 static inline void
4616 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4617 {
4618         struct thread *t = &p->threads[p->thread_id];
4619         struct instruction *ip = t->ip;
4620
4621         /* Structs. */
4622         __instr_regadd_rim_exec(p, t, ip);
4623
4624         /* Thread. */
4625         thread_ip_inc(p);
4626 }
4627
4628 static inline void
4629 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4630 {
4631         struct thread *t = &p->threads[p->thread_id];
4632         struct instruction *ip = t->ip;
4633
4634         /* Structs. */
4635         __instr_regadd_rii_exec(p, t, ip);
4636
4637         /* Thread. */
4638         thread_ip_inc(p);
4639 }
4640
4641 /*
4642  * metarray.
4643  */
4644 static struct metarray *
4645 metarray_find(struct rte_swx_pipeline *p, const char *name);
4646
4647 static int
4648 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4649                             struct action *action,
4650                             char **tokens,
4651                             int n_tokens,
4652                             struct instruction *instr,
4653                             struct instruction_data *data __rte_unused)
4654 {
4655         char *metarray = tokens[1], *idx = tokens[2];
4656         struct metarray *m;
4657         struct field *fidx;
4658         uint32_t idx_struct_id, idx_val;
4659
4660         CHECK(n_tokens == 3, EINVAL);
4661
4662         m = metarray_find(p, metarray);
4663         CHECK(m, EINVAL);
4664
4665         /* METPREFETCH_H, METPREFETCH_M. */
4666         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4667         if (fidx) {
4668                 CHECK(!fidx->var_size, EINVAL);
4669
4670                 instr->type = INSTR_METPREFETCH_M;
4671                 if (idx[0] == 'h')
4672                         instr->type = INSTR_METPREFETCH_H;
4673
4674                 instr->meter.metarray_id = m->id;
4675                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4676                 instr->meter.idx.n_bits = fidx->n_bits;
4677                 instr->meter.idx.offset = fidx->offset / 8;
4678                 return 0;
4679         }
4680
4681         /* METPREFETCH_I. */
4682         idx_val = strtoul(idx, &idx, 0);
4683         CHECK(!idx[0], EINVAL);
4684
4685         instr->type = INSTR_METPREFETCH_I;
4686         instr->meter.metarray_id = m->id;
4687         instr->meter.idx_val = idx_val;
4688         return 0;
4689 }
4690
4691 static int
4692 instr_meter_translate(struct rte_swx_pipeline *p,
4693                       struct action *action,
4694                       char **tokens,
4695                       int n_tokens,
4696                       struct instruction *instr,
4697                       struct instruction_data *data __rte_unused)
4698 {
4699         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4700         char *color_in = tokens[4], *color_out = tokens[5];
4701         struct metarray *m;
4702         struct field *fidx, *flength, *fcin, *fcout;
4703         uint32_t idx_struct_id, length_struct_id;
4704         uint32_t color_in_struct_id, color_out_struct_id;
4705
4706         CHECK(n_tokens == 6, EINVAL);
4707
4708         m = metarray_find(p, metarray);
4709         CHECK(m, EINVAL);
4710
4711         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4712
4713         flength = struct_field_parse(p, action, length, &length_struct_id);
4714         CHECK(flength, EINVAL);
4715         CHECK(!flength->var_size, EINVAL);
4716
4717         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4718
4719         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4720         CHECK(fcout, EINVAL);
4721         CHECK(!fcout->var_size, EINVAL);
4722
4723         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4724         if (fidx && fcin) {
4725                 CHECK(!fidx->var_size, EINVAL);
4726                 CHECK(!fcin->var_size, EINVAL);
4727
4728                 instr->type = INSTR_METER_MMM;
4729                 if (idx[0] == 'h' && length[0] == 'h')
4730                         instr->type = INSTR_METER_HHM;
4731                 if (idx[0] == 'h' && length[0] != 'h')
4732                         instr->type = INSTR_METER_HMM;
4733                 if (idx[0] != 'h' && length[0] == 'h')
4734                         instr->type = INSTR_METER_MHM;
4735
4736                 instr->meter.metarray_id = m->id;
4737
4738                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4739                 instr->meter.idx.n_bits = fidx->n_bits;
4740                 instr->meter.idx.offset = fidx->offset / 8;
4741
4742                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4743                 instr->meter.length.n_bits = flength->n_bits;
4744                 instr->meter.length.offset = flength->offset / 8;
4745
4746                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4747                 instr->meter.color_in.n_bits = fcin->n_bits;
4748                 instr->meter.color_in.offset = fcin->offset / 8;
4749
4750                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4751                 instr->meter.color_out.n_bits = fcout->n_bits;
4752                 instr->meter.color_out.offset = fcout->offset / 8;
4753         }
4754
4755         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4756         if (fidx && !fcin) {
4757                 uint32_t color_in_val;
4758
4759                 CHECK(!fidx->var_size, EINVAL);
4760
4761                 color_in_val = strtoul(color_in, &color_in, 0);
4762                 CHECK(!color_in[0], EINVAL);
4763
4764                 instr->type = INSTR_METER_MMI;
4765                 if (idx[0] == 'h' && length[0] == 'h')
4766                         instr->type = INSTR_METER_HHI;
4767                 if (idx[0] == 'h' && length[0] != 'h')
4768                         instr->type = INSTR_METER_HMI;
4769                 if (idx[0] != 'h' && length[0] == 'h')
4770                         instr->type = INSTR_METER_MHI;
4771
4772                 instr->meter.metarray_id = m->id;
4773
4774                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4775                 instr->meter.idx.n_bits = fidx->n_bits;
4776                 instr->meter.idx.offset = fidx->offset / 8;
4777
4778                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4779                 instr->meter.length.n_bits = flength->n_bits;
4780                 instr->meter.length.offset = flength->offset / 8;
4781
4782                 instr->meter.color_in_val = color_in_val;
4783
4784                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4785                 instr->meter.color_out.n_bits = fcout->n_bits;
4786                 instr->meter.color_out.offset = fcout->offset / 8;
4787         }
4788
4789         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4790         if (!fidx && fcin) {
4791                 uint32_t idx_val;
4792
4793                 idx_val = strtoul(idx, &idx, 0);
4794                 CHECK(!idx[0], EINVAL);
4795
4796                 CHECK(!fcin->var_size, EINVAL);
4797
4798                 instr->type = INSTR_METER_IMM;
4799                 if (length[0] == 'h')
4800                         instr->type = INSTR_METER_IHM;
4801
4802                 instr->meter.metarray_id = m->id;
4803
4804                 instr->meter.idx_val = idx_val;
4805
4806                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4807                 instr->meter.length.n_bits = flength->n_bits;
4808                 instr->meter.length.offset = flength->offset / 8;
4809
4810                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4811                 instr->meter.color_in.n_bits = fcin->n_bits;
4812                 instr->meter.color_in.offset = fcin->offset / 8;
4813
4814                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4815                 instr->meter.color_out.n_bits = fcout->n_bits;
4816                 instr->meter.color_out.offset = fcout->offset / 8;
4817         }
4818
4819         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4820         if (!fidx && !fcin) {
4821                 uint32_t idx_val, color_in_val;
4822
4823                 idx_val = strtoul(idx, &idx, 0);
4824                 CHECK(!idx[0], EINVAL);
4825
4826                 color_in_val = strtoul(color_in, &color_in, 0);
4827                 CHECK(!color_in[0], EINVAL);
4828
4829                 instr->type = INSTR_METER_IMI;
4830                 if (length[0] == 'h')
4831                         instr->type = INSTR_METER_IHI;
4832
4833                 instr->meter.metarray_id = m->id;
4834
4835                 instr->meter.idx_val = idx_val;
4836
4837                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4838                 instr->meter.length.n_bits = flength->n_bits;
4839                 instr->meter.length.offset = flength->offset / 8;
4840
4841                 instr->meter.color_in_val = color_in_val;
4842
4843                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4844                 instr->meter.color_out.n_bits = fcout->n_bits;
4845                 instr->meter.color_out.offset = fcout->offset / 8;
4846         }
4847
4848         return 0;
4849 }
4850
4851 static inline void
4852 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4853 {
4854         struct thread *t = &p->threads[p->thread_id];
4855         struct instruction *ip = t->ip;
4856
4857         /* Structs. */
4858         __instr_metprefetch_h_exec(p, t, ip);
4859
4860         /* Thread. */
4861         thread_ip_inc(p);
4862 }
4863
4864 static inline void
4865 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4866 {
4867         struct thread *t = &p->threads[p->thread_id];
4868         struct instruction *ip = t->ip;
4869
4870         /* Structs. */
4871         __instr_metprefetch_m_exec(p, t, ip);
4872
4873         /* Thread. */
4874         thread_ip_inc(p);
4875 }
4876
4877 static inline void
4878 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4879 {
4880         struct thread *t = &p->threads[p->thread_id];
4881         struct instruction *ip = t->ip;
4882
4883         /* Structs. */
4884         __instr_metprefetch_i_exec(p, t, ip);
4885
4886         /* Thread. */
4887         thread_ip_inc(p);
4888 }
4889
4890 static inline void
4891 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4892 {
4893         struct thread *t = &p->threads[p->thread_id];
4894         struct instruction *ip = t->ip;
4895
4896         /* Structs. */
4897         __instr_meter_hhm_exec(p, t, ip);
4898
4899         /* Thread. */
4900         thread_ip_inc(p);
4901 }
4902
4903 static inline void
4904 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4905 {
4906         struct thread *t = &p->threads[p->thread_id];
4907         struct instruction *ip = t->ip;
4908
4909         /* Structs. */
4910         __instr_meter_hhi_exec(p, t, ip);
4911
4912         /* Thread. */
4913         thread_ip_inc(p);
4914 }
4915
4916 static inline void
4917 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4918 {
4919         struct thread *t = &p->threads[p->thread_id];
4920         struct instruction *ip = t->ip;
4921
4922         /* Structs. */
4923         __instr_meter_hmm_exec(p, t, ip);
4924
4925         /* Thread. */
4926         thread_ip_inc(p);
4927 }
4928
4929 static inline void
4930 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4931 {
4932         struct thread *t = &p->threads[p->thread_id];
4933         struct instruction *ip = t->ip;
4934
4935         /* Structs. */
4936         __instr_meter_hmi_exec(p, t, ip);
4937
4938         /* Thread. */
4939         thread_ip_inc(p);
4940 }
4941
4942 static inline void
4943 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4944 {
4945         struct thread *t = &p->threads[p->thread_id];
4946         struct instruction *ip = t->ip;
4947
4948         /* Structs. */
4949         __instr_meter_mhm_exec(p, t, ip);
4950
4951         /* Thread. */
4952         thread_ip_inc(p);
4953 }
4954
4955 static inline void
4956 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4957 {
4958         struct thread *t = &p->threads[p->thread_id];
4959         struct instruction *ip = t->ip;
4960
4961         /* Structs. */
4962         __instr_meter_mhi_exec(p, t, ip);
4963
4964         /* Thread. */
4965         thread_ip_inc(p);
4966 }
4967
4968 static inline void
4969 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4970 {
4971         struct thread *t = &p->threads[p->thread_id];
4972         struct instruction *ip = t->ip;
4973
4974         /* Structs. */
4975         __instr_meter_mmm_exec(p, t, ip);
4976
4977         /* Thread. */
4978         thread_ip_inc(p);
4979 }
4980
4981 static inline void
4982 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4983 {
4984         struct thread *t = &p->threads[p->thread_id];
4985         struct instruction *ip = t->ip;
4986
4987         /* Structs. */
4988         __instr_meter_mmi_exec(p, t, ip);
4989
4990         /* Thread. */
4991         thread_ip_inc(p);
4992 }
4993
4994 static inline void
4995 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4996 {
4997         struct thread *t = &p->threads[p->thread_id];
4998         struct instruction *ip = t->ip;
4999
5000         /* Structs. */
5001         __instr_meter_ihm_exec(p, t, ip);
5002
5003         /* Thread. */
5004         thread_ip_inc(p);
5005 }
5006
5007 static inline void
5008 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5009 {
5010         struct thread *t = &p->threads[p->thread_id];
5011         struct instruction *ip = t->ip;
5012
5013         /* Structs. */
5014         __instr_meter_ihi_exec(p, t, ip);
5015
5016         /* Thread. */
5017         thread_ip_inc(p);
5018 }
5019
5020 static inline void
5021 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5022 {
5023         struct thread *t = &p->threads[p->thread_id];
5024         struct instruction *ip = t->ip;
5025
5026         /* Structs. */
5027         __instr_meter_imm_exec(p, t, ip);
5028
5029         /* Thread. */
5030         thread_ip_inc(p);
5031 }
5032
5033 static inline void
5034 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5035 {
5036         struct thread *t = &p->threads[p->thread_id];
5037         struct instruction *ip = t->ip;
5038
5039         /* Structs. */
5040         __instr_meter_imi_exec(p, t, ip);
5041
5042         /* Thread. */
5043         thread_ip_inc(p);
5044 }
5045
5046 /*
5047  * jmp.
5048  */
5049 static int
5050 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5051                     struct action *action __rte_unused,
5052                     char **tokens,
5053                     int n_tokens,
5054                     struct instruction *instr,
5055                     struct instruction_data *data)
5056 {
5057         CHECK(n_tokens == 2, EINVAL);
5058
5059         strcpy(data->jmp_label, tokens[1]);
5060
5061         instr->type = INSTR_JMP;
5062         instr->jmp.ip = NULL; /* Resolved later. */
5063         return 0;
5064 }
5065
5066 static int
5067 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5068                           struct action *action __rte_unused,
5069                           char **tokens,
5070                           int n_tokens,
5071                           struct instruction *instr,
5072                           struct instruction_data *data)
5073 {
5074         struct header *h;
5075
5076         CHECK(n_tokens == 3, EINVAL);
5077
5078         strcpy(data->jmp_label, tokens[1]);
5079
5080         h = header_parse(p, tokens[2]);
5081         CHECK(h, EINVAL);
5082
5083         instr->type = INSTR_JMP_VALID;
5084         instr->jmp.ip = NULL; /* Resolved later. */
5085         instr->jmp.header_id = h->id;
5086         return 0;
5087 }
5088
5089 static int
5090 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5091                             struct action *action __rte_unused,
5092                             char **tokens,
5093                             int n_tokens,
5094                             struct instruction *instr,
5095                             struct instruction_data *data)
5096 {
5097         struct header *h;
5098
5099         CHECK(n_tokens == 3, EINVAL);
5100
5101         strcpy(data->jmp_label, tokens[1]);
5102
5103         h = header_parse(p, tokens[2]);
5104         CHECK(h, EINVAL);
5105
5106         instr->type = INSTR_JMP_INVALID;
5107         instr->jmp.ip = NULL; /* Resolved later. */
5108         instr->jmp.header_id = h->id;
5109         return 0;
5110 }
5111
5112 static int
5113 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5114                         struct action *action,
5115                         char **tokens,
5116                         int n_tokens,
5117                         struct instruction *instr,
5118                         struct instruction_data *data)
5119 {
5120         CHECK(!action, EINVAL);
5121         CHECK(n_tokens == 2, EINVAL);
5122
5123         strcpy(data->jmp_label, tokens[1]);
5124
5125         instr->type = INSTR_JMP_HIT;
5126         instr->jmp.ip = NULL; /* Resolved later. */
5127         return 0;
5128 }
5129
5130 static int
5131 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5132                          struct action *action,
5133                          char **tokens,
5134                          int n_tokens,
5135                          struct instruction *instr,
5136                          struct instruction_data *data)
5137 {
5138         CHECK(!action, EINVAL);
5139         CHECK(n_tokens == 2, EINVAL);
5140
5141         strcpy(data->jmp_label, tokens[1]);
5142
5143         instr->type = INSTR_JMP_MISS;
5144         instr->jmp.ip = NULL; /* Resolved later. */
5145         return 0;
5146 }
5147
5148 static int
5149 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5150                                struct action *action,
5151                                char **tokens,
5152                                int n_tokens,
5153                                struct instruction *instr,
5154                                struct instruction_data *data)
5155 {
5156         struct action *a;
5157
5158         CHECK(!action, EINVAL);
5159         CHECK(n_tokens == 3, EINVAL);
5160
5161         strcpy(data->jmp_label, tokens[1]);
5162
5163         a = action_find(p, tokens[2]);
5164         CHECK(a, EINVAL);
5165
5166         instr->type = INSTR_JMP_ACTION_HIT;
5167         instr->jmp.ip = NULL; /* Resolved later. */
5168         instr->jmp.action_id = a->id;
5169         return 0;
5170 }
5171
5172 static int
5173 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5174                                 struct action *action,
5175                                 char **tokens,
5176                                 int n_tokens,
5177                                 struct instruction *instr,
5178                                 struct instruction_data *data)
5179 {
5180         struct action *a;
5181
5182         CHECK(!action, EINVAL);
5183         CHECK(n_tokens == 3, EINVAL);
5184
5185         strcpy(data->jmp_label, tokens[1]);
5186
5187         a = action_find(p, tokens[2]);
5188         CHECK(a, EINVAL);
5189
5190         instr->type = INSTR_JMP_ACTION_MISS;
5191         instr->jmp.ip = NULL; /* Resolved later. */
5192         instr->jmp.action_id = a->id;
5193         return 0;
5194 }
5195
5196 static int
5197 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5198                        struct action *action,
5199                        char **tokens,
5200                        int n_tokens,
5201                        struct instruction *instr,
5202                        struct instruction_data *data)
5203 {
5204         char *a = tokens[2], *b = tokens[3];
5205         struct field *fa, *fb;
5206         uint64_t b_val;
5207         uint32_t a_struct_id, b_struct_id;
5208
5209         CHECK(n_tokens == 4, EINVAL);
5210
5211         strcpy(data->jmp_label, tokens[1]);
5212
5213         fa = struct_field_parse(p, action, a, &a_struct_id);
5214         CHECK(fa, EINVAL);
5215         CHECK(!fa->var_size, EINVAL);
5216
5217         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5218         fb = struct_field_parse(p, action, b, &b_struct_id);
5219         if (fb) {
5220                 CHECK(!fb->var_size, EINVAL);
5221
5222                 instr->type = INSTR_JMP_EQ;
5223                 if (a[0] != 'h' && b[0] == 'h')
5224                         instr->type = INSTR_JMP_EQ_MH;
5225                 if (a[0] == 'h' && b[0] != 'h')
5226                         instr->type = INSTR_JMP_EQ_HM;
5227                 if (a[0] == 'h' && b[0] == 'h')
5228                         instr->type = INSTR_JMP_EQ_HH;
5229                 instr->jmp.ip = NULL; /* Resolved later. */
5230
5231                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5232                 instr->jmp.a.n_bits = fa->n_bits;
5233                 instr->jmp.a.offset = fa->offset / 8;
5234                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5235                 instr->jmp.b.n_bits = fb->n_bits;
5236                 instr->jmp.b.offset = fb->offset / 8;
5237                 return 0;
5238         }
5239
5240         /* JMP_EQ_I. */
5241         b_val = strtoull(b, &b, 0);
5242         CHECK(!b[0], EINVAL);
5243
5244         if (a[0] == 'h')
5245                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5246
5247         instr->type = INSTR_JMP_EQ_I;
5248         instr->jmp.ip = NULL; /* Resolved later. */
5249         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5250         instr->jmp.a.n_bits = fa->n_bits;
5251         instr->jmp.a.offset = fa->offset / 8;
5252         instr->jmp.b_val = b_val;
5253         return 0;
5254 }
5255
5256 static int
5257 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5258                         struct action *action,
5259                         char **tokens,
5260                         int n_tokens,
5261                         struct instruction *instr,
5262                         struct instruction_data *data)
5263 {
5264         char *a = tokens[2], *b = tokens[3];
5265         struct field *fa, *fb;
5266         uint64_t b_val;
5267         uint32_t a_struct_id, b_struct_id;
5268
5269         CHECK(n_tokens == 4, EINVAL);
5270
5271         strcpy(data->jmp_label, tokens[1]);
5272
5273         fa = struct_field_parse(p, action, a, &a_struct_id);
5274         CHECK(fa, EINVAL);
5275         CHECK(!fa->var_size, EINVAL);
5276
5277         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5278         fb = struct_field_parse(p, action, b, &b_struct_id);
5279         if (fb) {
5280                 CHECK(!fb->var_size, EINVAL);
5281
5282                 instr->type = INSTR_JMP_NEQ;
5283                 if (a[0] != 'h' && b[0] == 'h')
5284                         instr->type = INSTR_JMP_NEQ_MH;
5285                 if (a[0] == 'h' && b[0] != 'h')
5286                         instr->type = INSTR_JMP_NEQ_HM;
5287                 if (a[0] == 'h' && b[0] == 'h')
5288                         instr->type = INSTR_JMP_NEQ_HH;
5289                 instr->jmp.ip = NULL; /* Resolved later. */
5290
5291                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5292                 instr->jmp.a.n_bits = fa->n_bits;
5293                 instr->jmp.a.offset = fa->offset / 8;
5294                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5295                 instr->jmp.b.n_bits = fb->n_bits;
5296                 instr->jmp.b.offset = fb->offset / 8;
5297                 return 0;
5298         }
5299
5300         /* JMP_NEQ_I. */
5301         b_val = strtoull(b, &b, 0);
5302         CHECK(!b[0], EINVAL);
5303
5304         if (a[0] == 'h')
5305                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5306
5307         instr->type = INSTR_JMP_NEQ_I;
5308         instr->jmp.ip = NULL; /* Resolved later. */
5309         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5310         instr->jmp.a.n_bits = fa->n_bits;
5311         instr->jmp.a.offset = fa->offset / 8;
5312         instr->jmp.b_val = b_val;
5313         return 0;
5314 }
5315
5316 static int
5317 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5318                        struct action *action,
5319                        char **tokens,
5320                        int n_tokens,
5321                        struct instruction *instr,
5322                        struct instruction_data *data)
5323 {
5324         char *a = tokens[2], *b = tokens[3];
5325         struct field *fa, *fb;
5326         uint64_t b_val;
5327         uint32_t a_struct_id, b_struct_id;
5328
5329         CHECK(n_tokens == 4, EINVAL);
5330
5331         strcpy(data->jmp_label, tokens[1]);
5332
5333         fa = struct_field_parse(p, action, a, &a_struct_id);
5334         CHECK(fa, EINVAL);
5335         CHECK(!fa->var_size, EINVAL);
5336
5337         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5338         fb = struct_field_parse(p, action, b, &b_struct_id);
5339         if (fb) {
5340                 CHECK(!fb->var_size, EINVAL);
5341
5342                 instr->type = INSTR_JMP_LT;
5343                 if (a[0] == 'h' && b[0] != 'h')
5344                         instr->type = INSTR_JMP_LT_HM;
5345                 if (a[0] != 'h' && b[0] == 'h')
5346                         instr->type = INSTR_JMP_LT_MH;
5347                 if (a[0] == 'h' && b[0] == 'h')
5348                         instr->type = INSTR_JMP_LT_HH;
5349                 instr->jmp.ip = NULL; /* Resolved later. */
5350
5351                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5352                 instr->jmp.a.n_bits = fa->n_bits;
5353                 instr->jmp.a.offset = fa->offset / 8;
5354                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5355                 instr->jmp.b.n_bits = fb->n_bits;
5356                 instr->jmp.b.offset = fb->offset / 8;
5357                 return 0;
5358         }
5359
5360         /* JMP_LT_MI, JMP_LT_HI. */
5361         b_val = strtoull(b, &b, 0);
5362         CHECK(!b[0], EINVAL);
5363
5364         instr->type = INSTR_JMP_LT_MI;
5365         if (a[0] == 'h')
5366                 instr->type = INSTR_JMP_LT_HI;
5367         instr->jmp.ip = NULL; /* Resolved later. */
5368
5369         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5370         instr->jmp.a.n_bits = fa->n_bits;
5371         instr->jmp.a.offset = fa->offset / 8;
5372         instr->jmp.b_val = b_val;
5373         return 0;
5374 }
5375
5376 static int
5377 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5378                        struct action *action,
5379                        char **tokens,
5380                        int n_tokens,
5381                        struct instruction *instr,
5382                        struct instruction_data *data)
5383 {
5384         char *a = tokens[2], *b = tokens[3];
5385         struct field *fa, *fb;
5386         uint64_t b_val;
5387         uint32_t a_struct_id, b_struct_id;
5388
5389         CHECK(n_tokens == 4, EINVAL);
5390
5391         strcpy(data->jmp_label, tokens[1]);
5392
5393         fa = struct_field_parse(p, action, a, &a_struct_id);
5394         CHECK(fa, EINVAL);
5395         CHECK(!fa->var_size, EINVAL);
5396
5397         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5398         fb = struct_field_parse(p, action, b, &b_struct_id);
5399         if (fb) {
5400                 CHECK(!fb->var_size, EINVAL);
5401
5402                 instr->type = INSTR_JMP_GT;
5403                 if (a[0] == 'h' && b[0] != 'h')
5404                         instr->type = INSTR_JMP_GT_HM;
5405                 if (a[0] != 'h' && b[0] == 'h')
5406                         instr->type = INSTR_JMP_GT_MH;
5407                 if (a[0] == 'h' && b[0] == 'h')
5408                         instr->type = INSTR_JMP_GT_HH;
5409                 instr->jmp.ip = NULL; /* Resolved later. */
5410
5411                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5412                 instr->jmp.a.n_bits = fa->n_bits;
5413                 instr->jmp.a.offset = fa->offset / 8;
5414                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5415                 instr->jmp.b.n_bits = fb->n_bits;
5416                 instr->jmp.b.offset = fb->offset / 8;
5417                 return 0;
5418         }
5419
5420         /* JMP_GT_MI, JMP_GT_HI. */
5421         b_val = strtoull(b, &b, 0);
5422         CHECK(!b[0], EINVAL);
5423
5424         instr->type = INSTR_JMP_GT_MI;
5425         if (a[0] == 'h')
5426                 instr->type = INSTR_JMP_GT_HI;
5427         instr->jmp.ip = NULL; /* Resolved later. */
5428
5429         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5430         instr->jmp.a.n_bits = fa->n_bits;
5431         instr->jmp.a.offset = fa->offset / 8;
5432         instr->jmp.b_val = b_val;
5433         return 0;
5434 }
5435
5436 static inline void
5437 instr_jmp_exec(struct rte_swx_pipeline *p)
5438 {
5439         struct thread *t = &p->threads[p->thread_id];
5440         struct instruction *ip = t->ip;
5441
5442         TRACE("[Thread %2u] jmp\n", p->thread_id);
5443
5444         thread_ip_set(t, ip->jmp.ip);
5445 }
5446
5447 static inline void
5448 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5449 {
5450         struct thread *t = &p->threads[p->thread_id];
5451         struct instruction *ip = t->ip;
5452         uint32_t header_id = ip->jmp.header_id;
5453
5454         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5455
5456         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5457 }
5458
5459 static inline void
5460 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5461 {
5462         struct thread *t = &p->threads[p->thread_id];
5463         struct instruction *ip = t->ip;
5464         uint32_t header_id = ip->jmp.header_id;
5465
5466         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5467
5468         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5469 }
5470
5471 static inline void
5472 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5473 {
5474         struct thread *t = &p->threads[p->thread_id];
5475         struct instruction *ip = t->ip;
5476         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5477
5478         TRACE("[Thread %2u] jmph\n", p->thread_id);
5479
5480         t->ip = ip_next[t->hit];
5481 }
5482
5483 static inline void
5484 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5485 {
5486         struct thread *t = &p->threads[p->thread_id];
5487         struct instruction *ip = t->ip;
5488         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5489
5490         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5491
5492         t->ip = ip_next[t->hit];
5493 }
5494
5495 static inline void
5496 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5497 {
5498         struct thread *t = &p->threads[p->thread_id];
5499         struct instruction *ip = t->ip;
5500
5501         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5502
5503         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5504 }
5505
5506 static inline void
5507 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5508 {
5509         struct thread *t = &p->threads[p->thread_id];
5510         struct instruction *ip = t->ip;
5511
5512         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5513
5514         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5515 }
5516
5517 static inline void
5518 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5519 {
5520         struct thread *t = &p->threads[p->thread_id];
5521         struct instruction *ip = t->ip;
5522
5523         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5524
5525         JMP_CMP(t, ip, ==);
5526 }
5527
5528 static inline void
5529 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5530 {
5531         struct thread *t = &p->threads[p->thread_id];
5532         struct instruction *ip = t->ip;
5533
5534         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5535
5536         JMP_CMP_MH(t, ip, ==);
5537 }
5538
5539 static inline void
5540 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5541 {
5542         struct thread *t = &p->threads[p->thread_id];
5543         struct instruction *ip = t->ip;
5544
5545         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5546
5547         JMP_CMP_HM(t, ip, ==);
5548 }
5549
5550 static inline void
5551 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5552 {
5553         struct thread *t = &p->threads[p->thread_id];
5554         struct instruction *ip = t->ip;
5555
5556         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5557
5558         JMP_CMP_HH_FAST(t, ip, ==);
5559 }
5560
5561 static inline void
5562 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5563 {
5564         struct thread *t = &p->threads[p->thread_id];
5565         struct instruction *ip = t->ip;
5566
5567         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5568
5569         JMP_CMP_I(t, ip, ==);
5570 }
5571
5572 static inline void
5573 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5574 {
5575         struct thread *t = &p->threads[p->thread_id];
5576         struct instruction *ip = t->ip;
5577
5578         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5579
5580         JMP_CMP(t, ip, !=);
5581 }
5582
5583 static inline void
5584 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5585 {
5586         struct thread *t = &p->threads[p->thread_id];
5587         struct instruction *ip = t->ip;
5588
5589         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5590
5591         JMP_CMP_MH(t, ip, !=);
5592 }
5593
5594 static inline void
5595 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5596 {
5597         struct thread *t = &p->threads[p->thread_id];
5598         struct instruction *ip = t->ip;
5599
5600         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5601
5602         JMP_CMP_HM(t, ip, !=);
5603 }
5604
5605 static inline void
5606 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5607 {
5608         struct thread *t = &p->threads[p->thread_id];
5609         struct instruction *ip = t->ip;
5610
5611         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5612
5613         JMP_CMP_HH_FAST(t, ip, !=);
5614 }
5615
5616 static inline void
5617 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5618 {
5619         struct thread *t = &p->threads[p->thread_id];
5620         struct instruction *ip = t->ip;
5621
5622         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5623
5624         JMP_CMP_I(t, ip, !=);
5625 }
5626
5627 static inline void
5628 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5629 {
5630         struct thread *t = &p->threads[p->thread_id];
5631         struct instruction *ip = t->ip;
5632
5633         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5634
5635         JMP_CMP(t, ip, <);
5636 }
5637
5638 static inline void
5639 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5640 {
5641         struct thread *t = &p->threads[p->thread_id];
5642         struct instruction *ip = t->ip;
5643
5644         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5645
5646         JMP_CMP_MH(t, ip, <);
5647 }
5648
5649 static inline void
5650 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5651 {
5652         struct thread *t = &p->threads[p->thread_id];
5653         struct instruction *ip = t->ip;
5654
5655         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5656
5657         JMP_CMP_HM(t, ip, <);
5658 }
5659
5660 static inline void
5661 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5662 {
5663         struct thread *t = &p->threads[p->thread_id];
5664         struct instruction *ip = t->ip;
5665
5666         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5667
5668         JMP_CMP_HH(t, ip, <);
5669 }
5670
5671 static inline void
5672 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5673 {
5674         struct thread *t = &p->threads[p->thread_id];
5675         struct instruction *ip = t->ip;
5676
5677         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5678
5679         JMP_CMP_MI(t, ip, <);
5680 }
5681
5682 static inline void
5683 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5684 {
5685         struct thread *t = &p->threads[p->thread_id];
5686         struct instruction *ip = t->ip;
5687
5688         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5689
5690         JMP_CMP_HI(t, ip, <);
5691 }
5692
5693 static inline void
5694 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5695 {
5696         struct thread *t = &p->threads[p->thread_id];
5697         struct instruction *ip = t->ip;
5698
5699         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5700
5701         JMP_CMP(t, ip, >);
5702 }
5703
5704 static inline void
5705 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5706 {
5707         struct thread *t = &p->threads[p->thread_id];
5708         struct instruction *ip = t->ip;
5709
5710         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5711
5712         JMP_CMP_MH(t, ip, >);
5713 }
5714
5715 static inline void
5716 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5717 {
5718         struct thread *t = &p->threads[p->thread_id];
5719         struct instruction *ip = t->ip;
5720
5721         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5722
5723         JMP_CMP_HM(t, ip, >);
5724 }
5725
5726 static inline void
5727 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5728 {
5729         struct thread *t = &p->threads[p->thread_id];
5730         struct instruction *ip = t->ip;
5731
5732         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5733
5734         JMP_CMP_HH(t, ip, >);
5735 }
5736
5737 static inline void
5738 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5739 {
5740         struct thread *t = &p->threads[p->thread_id];
5741         struct instruction *ip = t->ip;
5742
5743         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5744
5745         JMP_CMP_MI(t, ip, >);
5746 }
5747
5748 static inline void
5749 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5750 {
5751         struct thread *t = &p->threads[p->thread_id];
5752         struct instruction *ip = t->ip;
5753
5754         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5755
5756         JMP_CMP_HI(t, ip, >);
5757 }
5758
5759 /*
5760  * return.
5761  */
5762 static int
5763 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5764                        struct action *action,
5765                        char **tokens __rte_unused,
5766                        int n_tokens,
5767                        struct instruction *instr,
5768                        struct instruction_data *data __rte_unused)
5769 {
5770         CHECK(action, EINVAL);
5771         CHECK(n_tokens == 1, EINVAL);
5772
5773         instr->type = INSTR_RETURN;
5774         return 0;
5775 }
5776
5777 static inline void
5778 instr_return_exec(struct rte_swx_pipeline *p)
5779 {
5780         struct thread *t = &p->threads[p->thread_id];
5781
5782         TRACE("[Thread %2u] return\n", p->thread_id);
5783
5784         t->ip = t->ret;
5785 }
5786
5787 static int
5788 instr_translate(struct rte_swx_pipeline *p,
5789                 struct action *action,
5790                 char *string,
5791                 struct instruction *instr,
5792                 struct instruction_data *data)
5793 {
5794         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5795         int n_tokens = 0, tpos = 0;
5796
5797         /* Parse the instruction string into tokens. */
5798         for ( ; ; ) {
5799                 char *token;
5800
5801                 token = strtok_r(string, " \t\v", &string);
5802                 if (!token)
5803                         break;
5804
5805                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5806                 CHECK_NAME(token, EINVAL);
5807
5808                 tokens[n_tokens] = token;
5809                 n_tokens++;
5810         }
5811
5812         CHECK(n_tokens, EINVAL);
5813
5814         /* Handle the optional instruction label. */
5815         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5816                 strcpy(data->label, tokens[0]);
5817
5818                 tpos += 2;
5819                 CHECK(n_tokens - tpos, EINVAL);
5820         }
5821
5822         /* Identify the instruction type. */
5823         if (!strcmp(tokens[tpos], "rx"))
5824                 return instr_rx_translate(p,
5825                                           action,
5826                                           &tokens[tpos],
5827                                           n_tokens - tpos,
5828                                           instr,
5829                                           data);
5830
5831         if (!strcmp(tokens[tpos], "tx"))
5832                 return instr_tx_translate(p,
5833                                           action,
5834                                           &tokens[tpos],
5835                                           n_tokens - tpos,
5836                                           instr,
5837                                           data);
5838
5839         if (!strcmp(tokens[tpos], "drop"))
5840                 return instr_drop_translate(p,
5841                                             action,
5842                                             &tokens[tpos],
5843                                             n_tokens - tpos,
5844                                             instr,
5845                                             data);
5846
5847         if (!strcmp(tokens[tpos], "mirror"))
5848                 return instr_mirror_translate(p,
5849                                               action,
5850                                               &tokens[tpos],
5851                                               n_tokens - tpos,
5852                                               instr,
5853                                               data);
5854
5855         if (!strcmp(tokens[tpos], "recirculate"))
5856                 return instr_recirculate_translate(p,
5857                                               action,
5858                                               &tokens[tpos],
5859                                               n_tokens - tpos,
5860                                               instr,
5861                                               data);
5862
5863         if (!strcmp(tokens[tpos], "recircid"))
5864                 return instr_recircid_translate(p,
5865                                               action,
5866                                               &tokens[tpos],
5867                                               n_tokens - tpos,
5868                                               instr,
5869                                               data);
5870
5871         if (!strcmp(tokens[tpos], "extract"))
5872                 return instr_hdr_extract_translate(p,
5873                                                    action,
5874                                                    &tokens[tpos],
5875                                                    n_tokens - tpos,
5876                                                    instr,
5877                                                    data);
5878
5879         if (!strcmp(tokens[tpos], "lookahead"))
5880                 return instr_hdr_lookahead_translate(p,
5881                                                      action,
5882                                                      &tokens[tpos],
5883                                                      n_tokens - tpos,
5884                                                      instr,
5885                                                      data);
5886
5887         if (!strcmp(tokens[tpos], "emit"))
5888                 return instr_hdr_emit_translate(p,
5889                                                 action,
5890                                                 &tokens[tpos],
5891                                                 n_tokens - tpos,
5892                                                 instr,
5893                                                 data);
5894
5895         if (!strcmp(tokens[tpos], "validate"))
5896                 return instr_hdr_validate_translate(p,
5897                                                     action,
5898                                                     &tokens[tpos],
5899                                                     n_tokens - tpos,
5900                                                     instr,
5901                                                     data);
5902
5903         if (!strcmp(tokens[tpos], "invalidate"))
5904                 return instr_hdr_invalidate_translate(p,
5905                                                       action,
5906                                                       &tokens[tpos],
5907                                                       n_tokens - tpos,
5908                                                       instr,
5909                                                       data);
5910
5911         if (!strcmp(tokens[tpos], "mov"))
5912                 return instr_mov_translate(p,
5913                                            action,
5914                                            &tokens[tpos],
5915                                            n_tokens - tpos,
5916                                            instr,
5917                                            data);
5918
5919         if (!strcmp(tokens[tpos], "add"))
5920                 return instr_alu_add_translate(p,
5921                                                action,
5922                                                &tokens[tpos],
5923                                                n_tokens - tpos,
5924                                                instr,
5925                                                data);
5926
5927         if (!strcmp(tokens[tpos], "sub"))
5928                 return instr_alu_sub_translate(p,
5929                                                action,
5930                                                &tokens[tpos],
5931                                                n_tokens - tpos,
5932                                                instr,
5933                                                data);
5934
5935         if (!strcmp(tokens[tpos], "ckadd"))
5936                 return instr_alu_ckadd_translate(p,
5937                                                  action,
5938                                                  &tokens[tpos],
5939                                                  n_tokens - tpos,
5940                                                  instr,
5941                                                  data);
5942
5943         if (!strcmp(tokens[tpos], "cksub"))
5944                 return instr_alu_cksub_translate(p,
5945                                                  action,
5946                                                  &tokens[tpos],
5947                                                  n_tokens - tpos,
5948                                                  instr,
5949                                                  data);
5950
5951         if (!strcmp(tokens[tpos], "and"))
5952                 return instr_alu_and_translate(p,
5953                                                action,
5954                                                &tokens[tpos],
5955                                                n_tokens - tpos,
5956                                                instr,
5957                                                data);
5958
5959         if (!strcmp(tokens[tpos], "or"))
5960                 return instr_alu_or_translate(p,
5961                                               action,
5962                                               &tokens[tpos],
5963                                               n_tokens - tpos,
5964                                               instr,
5965                                               data);
5966
5967         if (!strcmp(tokens[tpos], "xor"))
5968                 return instr_alu_xor_translate(p,
5969                                                action,
5970                                                &tokens[tpos],
5971                                                n_tokens - tpos,
5972                                                instr,
5973                                                data);
5974
5975         if (!strcmp(tokens[tpos], "shl"))
5976                 return instr_alu_shl_translate(p,
5977                                                action,
5978                                                &tokens[tpos],
5979                                                n_tokens - tpos,
5980                                                instr,
5981                                                data);
5982
5983         if (!strcmp(tokens[tpos], "shr"))
5984                 return instr_alu_shr_translate(p,
5985                                                action,
5986                                                &tokens[tpos],
5987                                                n_tokens - tpos,
5988                                                instr,
5989                                                data);
5990
5991         if (!strcmp(tokens[tpos], "regprefetch"))
5992                 return instr_regprefetch_translate(p,
5993                                                    action,
5994                                                    &tokens[tpos],
5995                                                    n_tokens - tpos,
5996                                                    instr,
5997                                                    data);
5998
5999         if (!strcmp(tokens[tpos], "regrd"))
6000                 return instr_regrd_translate(p,
6001                                              action,
6002                                              &tokens[tpos],
6003                                              n_tokens - tpos,
6004                                              instr,
6005                                              data);
6006
6007         if (!strcmp(tokens[tpos], "regwr"))
6008                 return instr_regwr_translate(p,
6009                                              action,
6010                                              &tokens[tpos],
6011                                              n_tokens - tpos,
6012                                              instr,
6013                                              data);
6014
6015         if (!strcmp(tokens[tpos], "regadd"))
6016                 return instr_regadd_translate(p,
6017                                               action,
6018                                               &tokens[tpos],
6019                                               n_tokens - tpos,
6020                                               instr,
6021                                               data);
6022
6023         if (!strcmp(tokens[tpos], "metprefetch"))
6024                 return instr_metprefetch_translate(p,
6025                                                    action,
6026                                                    &tokens[tpos],
6027                                                    n_tokens - tpos,
6028                                                    instr,
6029                                                    data);
6030
6031         if (!strcmp(tokens[tpos], "meter"))
6032                 return instr_meter_translate(p,
6033                                              action,
6034                                              &tokens[tpos],
6035                                              n_tokens - tpos,
6036                                              instr,
6037                                              data);
6038
6039         if (!strcmp(tokens[tpos], "table"))
6040                 return instr_table_translate(p,
6041                                              action,
6042                                              &tokens[tpos],
6043                                              n_tokens - tpos,
6044                                              instr,
6045                                              data);
6046
6047         if (!strcmp(tokens[tpos], "learn"))
6048                 return instr_learn_translate(p,
6049                                              action,
6050                                              &tokens[tpos],
6051                                              n_tokens - tpos,
6052                                              instr,
6053                                              data);
6054
6055         if (!strcmp(tokens[tpos], "forget"))
6056                 return instr_forget_translate(p,
6057                                               action,
6058                                               &tokens[tpos],
6059                                               n_tokens - tpos,
6060                                               instr,
6061                                               data);
6062
6063         if (!strcmp(tokens[tpos], "extern"))
6064                 return instr_extern_translate(p,
6065                                               action,
6066                                               &tokens[tpos],
6067                                               n_tokens - tpos,
6068                                               instr,
6069                                               data);
6070
6071         if (!strcmp(tokens[tpos], "jmp"))
6072                 return instr_jmp_translate(p,
6073                                            action,
6074                                            &tokens[tpos],
6075                                            n_tokens - tpos,
6076                                            instr,
6077                                            data);
6078
6079         if (!strcmp(tokens[tpos], "jmpv"))
6080                 return instr_jmp_valid_translate(p,
6081                                                  action,
6082                                                  &tokens[tpos],
6083                                                  n_tokens - tpos,
6084                                                  instr,
6085                                                  data);
6086
6087         if (!strcmp(tokens[tpos], "jmpnv"))
6088                 return instr_jmp_invalid_translate(p,
6089                                                    action,
6090                                                    &tokens[tpos],
6091                                                    n_tokens - tpos,
6092                                                    instr,
6093                                                    data);
6094
6095         if (!strcmp(tokens[tpos], "jmph"))
6096                 return instr_jmp_hit_translate(p,
6097                                                action,
6098                                                &tokens[tpos],
6099                                                n_tokens - tpos,
6100                                                instr,
6101                                                data);
6102
6103         if (!strcmp(tokens[tpos], "jmpnh"))
6104                 return instr_jmp_miss_translate(p,
6105                                                 action,
6106                                                 &tokens[tpos],
6107                                                 n_tokens - tpos,
6108                                                 instr,
6109                                                 data);
6110
6111         if (!strcmp(tokens[tpos], "jmpa"))
6112                 return instr_jmp_action_hit_translate(p,
6113                                                       action,
6114                                                       &tokens[tpos],
6115                                                       n_tokens - tpos,
6116                                                       instr,
6117                                                       data);
6118
6119         if (!strcmp(tokens[tpos], "jmpna"))
6120                 return instr_jmp_action_miss_translate(p,
6121                                                        action,
6122                                                        &tokens[tpos],
6123                                                        n_tokens - tpos,
6124                                                        instr,
6125                                                        data);
6126
6127         if (!strcmp(tokens[tpos], "jmpeq"))
6128                 return instr_jmp_eq_translate(p,
6129                                               action,
6130                                               &tokens[tpos],
6131                                               n_tokens - tpos,
6132                                               instr,
6133                                               data);
6134
6135         if (!strcmp(tokens[tpos], "jmpneq"))
6136                 return instr_jmp_neq_translate(p,
6137                                                action,
6138                                                &tokens[tpos],
6139                                                n_tokens - tpos,
6140                                                instr,
6141                                                data);
6142
6143         if (!strcmp(tokens[tpos], "jmplt"))
6144                 return instr_jmp_lt_translate(p,
6145                                               action,
6146                                               &tokens[tpos],
6147                                               n_tokens - tpos,
6148                                               instr,
6149                                               data);
6150
6151         if (!strcmp(tokens[tpos], "jmpgt"))
6152                 return instr_jmp_gt_translate(p,
6153                                               action,
6154                                               &tokens[tpos],
6155                                               n_tokens - tpos,
6156                                               instr,
6157                                               data);
6158
6159         if (!strcmp(tokens[tpos], "return"))
6160                 return instr_return_translate(p,
6161                                               action,
6162                                               &tokens[tpos],
6163                                               n_tokens - tpos,
6164                                               instr,
6165                                               data);
6166
6167         return -EINVAL;
6168 }
6169
6170 static struct instruction_data *
6171 label_find(struct instruction_data *data, uint32_t n, const char *label)
6172 {
6173         uint32_t i;
6174
6175         for (i = 0; i < n; i++)
6176                 if (!strcmp(label, data[i].label))
6177                         return &data[i];
6178
6179         return NULL;
6180 }
6181
6182 static uint32_t
6183 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6184 {
6185         uint32_t count = 0, i;
6186
6187         if (!label[0])
6188                 return 0;
6189
6190         for (i = 0; i < n; i++)
6191                 if (!strcmp(label, data[i].jmp_label))
6192                         count++;
6193
6194         return count;
6195 }
6196
6197 static int
6198 instr_label_check(struct instruction_data *instruction_data,
6199                   uint32_t n_instructions)
6200 {
6201         uint32_t i;
6202
6203         /* Check that all instruction labels are unique. */
6204         for (i = 0; i < n_instructions; i++) {
6205                 struct instruction_data *data = &instruction_data[i];
6206                 char *label = data->label;
6207                 uint32_t j;
6208
6209                 if (!label[0])
6210                         continue;
6211
6212                 for (j = i + 1; j < n_instructions; j++)
6213                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6214         }
6215
6216         /* Check that no jump instruction (either conditional or not) can jump to itself (loop). */
6217         for (i = 0; i < n_instructions; i++) {
6218                 struct instruction_data *data = &instruction_data[i];
6219                 char *label = data->label;
6220                 char *jmp_label = data->jmp_label;
6221
6222                 /* Continue if this instruction does not have a label or it is not a jump. */
6223                 if (!label[0] || !jmp_label[0])
6224                         continue;
6225
6226                 CHECK(strcmp(label, jmp_label), EINVAL);
6227         }
6228
6229         /* Get users for each instruction label. */
6230         for (i = 0; i < n_instructions; i++) {
6231                 struct instruction_data *data = &instruction_data[i];
6232                 char *label = data->label;
6233
6234                 data->n_users = label_is_used(instruction_data,
6235                                               n_instructions,
6236                                               label);
6237         }
6238
6239         return 0;
6240 }
6241
6242 static int
6243 instr_jmp_resolve(struct instruction *instructions,
6244                   struct instruction_data *instruction_data,
6245                   uint32_t n_instructions)
6246 {
6247         uint32_t i;
6248
6249         for (i = 0; i < n_instructions; i++) {
6250                 struct instruction *instr = &instructions[i];
6251                 struct instruction_data *data = &instruction_data[i];
6252                 struct instruction_data *found;
6253
6254                 if (!instruction_is_jmp(instr))
6255                         continue;
6256
6257                 found = label_find(instruction_data,
6258                                    n_instructions,
6259                                    data->jmp_label);
6260                 CHECK(found, EINVAL);
6261
6262                 instr->jmp.ip = &instructions[found - instruction_data];
6263         }
6264
6265         return 0;
6266 }
6267
6268 static int
6269 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6270              struct action *a,
6271              struct instruction *instr,
6272              struct instruction_data *data __rte_unused,
6273              uint32_t n_instructions)
6274 {
6275         if (!a) {
6276                 enum instruction_type type;
6277                 uint32_t i;
6278
6279                 /* Check that the first instruction is rx. */
6280                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6281
6282                 /* Check that there is at least one tx instruction. */
6283                 for (i = 0; i < n_instructions; i++) {
6284                         type = instr[i].type;
6285
6286                         if (instruction_is_tx(type))
6287                                 break;
6288                 }
6289                 CHECK(i < n_instructions, EINVAL);
6290
6291                 /* Check that the last instruction is either tx or unconditional
6292                  * jump.
6293                  */
6294                 type = instr[n_instructions - 1].type;
6295                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6296         }
6297
6298         if (a) {
6299                 enum instruction_type type;
6300                 uint32_t i;
6301
6302                 /* Check that there is at least one return or tx instruction. */
6303                 for (i = 0; i < n_instructions; i++) {
6304                         type = instr[i].type;
6305
6306                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6307                                 break;
6308                 }
6309                 CHECK(i < n_instructions, EINVAL);
6310         }
6311
6312         return 0;
6313 }
6314
6315 static uint32_t
6316 instr_compact(struct instruction *instructions,
6317               struct instruction_data *instruction_data,
6318               uint32_t n_instructions)
6319 {
6320         uint32_t i, pos = 0;
6321
6322         /* Eliminate the invalid instructions that have been optimized out. */
6323         for (i = 0; i < n_instructions; i++) {
6324                 struct instruction *instr = &instructions[i];
6325                 struct instruction_data *data = &instruction_data[i];
6326
6327                 if (data->invalid)
6328                         continue;
6329
6330                 if (i != pos) {
6331                         memcpy(&instructions[pos], instr, sizeof(*instr));
6332                         memcpy(&instruction_data[pos], data, sizeof(*data));
6333                 }
6334
6335                 pos++;
6336         }
6337
6338         return pos;
6339 }
6340
6341 static int
6342 instr_pattern_extract_many_search(struct instruction *instr,
6343                                   struct instruction_data *data,
6344                                   uint32_t n_instr,
6345                                   uint32_t *n_pattern_instr)
6346 {
6347         uint32_t i;
6348
6349         for (i = 0; i < n_instr; i++) {
6350                 if (data[i].invalid)
6351                         break;
6352
6353                 if (instr[i].type != INSTR_HDR_EXTRACT)
6354                         break;
6355
6356                 if (i == RTE_DIM(instr->io.hdr.header_id))
6357                         break;
6358
6359                 if (i && data[i].n_users)
6360                         break;
6361         }
6362
6363         if (i < 2)
6364                 return 0;
6365
6366         *n_pattern_instr = i;
6367         return 1;
6368 }
6369
6370 static void
6371 instr_pattern_extract_many_replace(struct instruction *instr,
6372                                    struct instruction_data *data,
6373                                    uint32_t n_instr)
6374 {
6375         uint32_t i;
6376
6377         for (i = 1; i < n_instr; i++) {
6378                 instr[0].type++;
6379                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6380                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6381                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6382
6383                 data[i].invalid = 1;
6384         }
6385 }
6386
6387 static uint32_t
6388 instr_pattern_extract_many_optimize(struct instruction *instructions,
6389                                     struct instruction_data *instruction_data,
6390                                     uint32_t n_instructions)
6391 {
6392         uint32_t i;
6393
6394         for (i = 0; i < n_instructions; ) {
6395                 struct instruction *instr = &instructions[i];
6396                 struct instruction_data *data = &instruction_data[i];
6397                 uint32_t n_instr = 0;
6398                 int detected;
6399
6400                 /* Extract many. */
6401                 detected = instr_pattern_extract_many_search(instr,
6402                                                              data,
6403                                                              n_instructions - i,
6404                                                              &n_instr);
6405                 if (detected) {
6406                         instr_pattern_extract_many_replace(instr,
6407                                                            data,
6408                                                            n_instr);
6409                         i += n_instr;
6410                         continue;
6411                 }
6412
6413                 /* No pattern starting at the current instruction. */
6414                 i++;
6415         }
6416
6417         /* Eliminate the invalid instructions that have been optimized out. */
6418         n_instructions = instr_compact(instructions,
6419                                        instruction_data,
6420                                        n_instructions);
6421
6422         return n_instructions;
6423 }
6424
6425 static int
6426 instr_pattern_emit_many_tx_search(struct instruction *instr,
6427                                   struct instruction_data *data,
6428                                   uint32_t n_instr,
6429                                   uint32_t *n_pattern_instr)
6430 {
6431         uint32_t i;
6432
6433         for (i = 0; i < n_instr; i++) {
6434                 if (data[i].invalid)
6435                         break;
6436
6437                 if (instr[i].type != INSTR_HDR_EMIT)
6438                         break;
6439
6440                 if (i == RTE_DIM(instr->io.hdr.header_id))
6441                         break;
6442
6443                 if (i && data[i].n_users)
6444                         break;
6445         }
6446
6447         if (!i)
6448                 return 0;
6449
6450         if (instr[i].type != INSTR_TX)
6451                 return 0;
6452
6453         if (data[i].n_users)
6454                 return 0;
6455
6456         i++;
6457
6458         *n_pattern_instr = i;
6459         return 1;
6460 }
6461
6462 static void
6463 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6464                                    struct instruction_data *data,
6465                                    uint32_t n_instr)
6466 {
6467         uint32_t i;
6468
6469         /* Any emit instruction in addition to the first one. */
6470         for (i = 1; i < n_instr - 1; i++) {
6471                 instr[0].type++;
6472                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6473                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6474                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6475
6476                 data[i].invalid = 1;
6477         }
6478
6479         /* The TX instruction is the last one in the pattern. */
6480         instr[0].type++;
6481         instr[0].io.io.offset = instr[i].io.io.offset;
6482         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6483         data[i].invalid = 1;
6484 }
6485
6486 static uint32_t
6487 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6488                                     struct instruction_data *instruction_data,
6489                                     uint32_t n_instructions)
6490 {
6491         uint32_t i;
6492
6493         for (i = 0; i < n_instructions; ) {
6494                 struct instruction *instr = &instructions[i];
6495                 struct instruction_data *data = &instruction_data[i];
6496                 uint32_t n_instr = 0;
6497                 int detected;
6498
6499                 /* Emit many + TX. */
6500                 detected = instr_pattern_emit_many_tx_search(instr,
6501                                                              data,
6502                                                              n_instructions - i,
6503                                                              &n_instr);
6504                 if (detected) {
6505                         instr_pattern_emit_many_tx_replace(instr,
6506                                                            data,
6507                                                            n_instr);
6508                         i += n_instr;
6509                         continue;
6510                 }
6511
6512                 /* No pattern starting at the current instruction. */
6513                 i++;
6514         }
6515
6516         /* Eliminate the invalid instructions that have been optimized out. */
6517         n_instructions = instr_compact(instructions,
6518                                        instruction_data,
6519                                        n_instructions);
6520
6521         return n_instructions;
6522 }
6523
6524 static uint32_t
6525 action_arg_src_mov_count(struct action *a,
6526                          uint32_t arg_id,
6527                          struct instruction *instructions,
6528                          struct instruction_data *instruction_data,
6529                          uint32_t n_instructions);
6530
6531 static int
6532 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6533                                       struct action *a,
6534                                       struct instruction *instr,
6535                                       struct instruction_data *data,
6536                                       uint32_t n_instr,
6537                                       struct instruction *instructions,
6538                                       struct instruction_data *instruction_data,
6539                                       uint32_t n_instructions,
6540                                       uint32_t *n_pattern_instr)
6541 {
6542         struct header *h;
6543         uint32_t src_field_id, i, j;
6544
6545         /* Prerequisites. */
6546         if (!a || !a->st)
6547                 return 0;
6548
6549         /* First instruction: MOV_HM. */
6550         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6551                 return 0;
6552
6553         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6554         if (!h || h->st->var_size)
6555                 return 0;
6556
6557         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6558                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6559                         break;
6560
6561         if (src_field_id == a->st->n_fields)
6562                 return 0;
6563
6564         if (instr[0].mov.dst.offset ||
6565             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6566             instr[0].mov.src.struct_id ||
6567             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6568             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6569                 return 0;
6570
6571         if ((n_instr < h->st->n_fields + 1) ||
6572              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6573                 return 0;
6574
6575         /* Subsequent instructions: MOV_HM. */
6576         for (i = 1; i < h->st->n_fields; i++)
6577                 if (data[i].invalid ||
6578                     data[i].n_users ||
6579                     (instr[i].type != INSTR_MOV_HM) ||
6580                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6581                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6582                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6583                     instr[i].mov.src.struct_id ||
6584                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6585                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6586                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6587                         return 0;
6588
6589         /* Last instruction: HDR_VALIDATE. */
6590         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6591             (instr[i].valid.header_id != h->id))
6592                 return 0;
6593
6594         /* Check that none of the action args that are used as source for this
6595          * DMA transfer are not used as source in any other mov instruction.
6596          */
6597         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6598                 uint32_t n_users;
6599
6600                 n_users = action_arg_src_mov_count(a,
6601                                                    j,
6602                                                    instructions,
6603                                                    instruction_data,
6604                                                    n_instructions);
6605                 if (n_users > 1)
6606                         return 0;
6607         }
6608
6609         *n_pattern_instr = 1 + i;
6610         return 1;
6611 }
6612
6613 static void
6614 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6615                                        struct action *a,
6616                                        struct instruction *instr,
6617                                        struct instruction_data *data,
6618                                        uint32_t n_instr)
6619 {
6620         struct header *h;
6621         uint32_t src_field_id, src_offset, i;
6622
6623         /* Read from the instructions before they are modified. */
6624         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6625         if (!h)
6626                 return;
6627
6628         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6629                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6630                         break;
6631
6632         if (src_field_id == a->st->n_fields)
6633                 return;
6634
6635         src_offset = instr[0].mov.src.offset;
6636
6637         /* Modify the instructions. */
6638         instr[0].type = INSTR_DMA_HT;
6639         instr[0].dma.dst.header_id[0] = h->id;
6640         instr[0].dma.dst.struct_id[0] = h->struct_id;
6641         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6642         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6643
6644         for (i = 1; i < n_instr; i++)
6645                 data[i].invalid = 1;
6646
6647         /* Update the endianness of the action arguments to header endianness. */
6648         for (i = 0; i < h->st->n_fields; i++)
6649                 a->args_endianness[src_field_id + i] = 1;
6650 }
6651
6652 static uint32_t
6653 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6654                                         struct action *a,
6655                                         struct instruction *instructions,
6656                                         struct instruction_data *instruction_data,
6657                                         uint32_t n_instructions)
6658 {
6659         uint32_t i;
6660
6661         if (!a || !a->st)
6662                 return n_instructions;
6663
6664         for (i = 0; i < n_instructions; ) {
6665                 struct instruction *instr = &instructions[i];
6666                 struct instruction_data *data = &instruction_data[i];
6667                 uint32_t n_instr = 0;
6668                 int detected;
6669
6670                 /* Mov all + validate. */
6671                 detected = instr_pattern_mov_all_validate_search(p,
6672                                                                  a,
6673                                                                  instr,
6674                                                                  data,
6675                                                                  n_instructions - i,
6676                                                                  instructions,
6677                                                                  instruction_data,
6678                                                                  n_instructions,
6679                                                                  &n_instr);
6680                 if (detected) {
6681                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6682                         i += n_instr;
6683                         continue;
6684                 }
6685
6686                 /* No pattern starting at the current instruction. */
6687                 i++;
6688         }
6689
6690         /* Eliminate the invalid instructions that have been optimized out. */
6691         n_instructions = instr_compact(instructions,
6692                                        instruction_data,
6693                                        n_instructions);
6694
6695         return n_instructions;
6696 }
6697
6698 static int
6699 instr_pattern_dma_many_search(struct instruction *instr,
6700                               struct instruction_data *data,
6701                               uint32_t n_instr,
6702                               uint32_t *n_pattern_instr)
6703 {
6704         uint32_t i;
6705
6706         for (i = 0; i < n_instr; i++) {
6707                 if (data[i].invalid)
6708                         break;
6709
6710                 if (instr[i].type != INSTR_DMA_HT)
6711                         break;
6712
6713                 if (i == RTE_DIM(instr->dma.dst.header_id))
6714                         break;
6715
6716                 if (i && data[i].n_users)
6717                         break;
6718         }
6719
6720         if (i < 2)
6721                 return 0;
6722
6723         *n_pattern_instr = i;
6724         return 1;
6725 }
6726
6727 static void
6728 instr_pattern_dma_many_replace(struct instruction *instr,
6729                                struct instruction_data *data,
6730                                uint32_t n_instr)
6731 {
6732         uint32_t i;
6733
6734         for (i = 1; i < n_instr; i++) {
6735                 instr[0].type++;
6736                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6737                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6738                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6739                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6740
6741                 data[i].invalid = 1;
6742         }
6743 }
6744
6745 static uint32_t
6746 instr_pattern_dma_many_optimize(struct instruction *instructions,
6747                struct instruction_data *instruction_data,
6748                uint32_t n_instructions)
6749 {
6750         uint32_t i;
6751
6752         for (i = 0; i < n_instructions; ) {
6753                 struct instruction *instr = &instructions[i];
6754                 struct instruction_data *data = &instruction_data[i];
6755                 uint32_t n_instr = 0;
6756                 int detected;
6757
6758                 /* DMA many. */
6759                 detected = instr_pattern_dma_many_search(instr,
6760                                                          data,
6761                                                          n_instructions - i,
6762                                                          &n_instr);
6763                 if (detected) {
6764                         instr_pattern_dma_many_replace(instr, data, n_instr);
6765                         i += n_instr;
6766                         continue;
6767                 }
6768
6769                 /* No pattern starting at the current instruction. */
6770                 i++;
6771         }
6772
6773         /* Eliminate the invalid instructions that have been optimized out. */
6774         n_instructions = instr_compact(instructions,
6775                                        instruction_data,
6776                                        n_instructions);
6777
6778         return n_instructions;
6779 }
6780
6781 static uint32_t
6782 instr_optimize(struct rte_swx_pipeline *p,
6783                struct action *a,
6784                struct instruction *instructions,
6785                struct instruction_data *instruction_data,
6786                uint32_t n_instructions)
6787 {
6788         /* Extract many. */
6789         n_instructions = instr_pattern_extract_many_optimize(instructions,
6790                                                              instruction_data,
6791                                                              n_instructions);
6792
6793         /* Emit many + TX. */
6794         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6795                                                              instruction_data,
6796                                                              n_instructions);
6797
6798         /* Mov all + validate. */
6799         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6800                                                                  a,
6801                                                                  instructions,
6802                                                                  instruction_data,
6803                                                                  n_instructions);
6804
6805         /* DMA many. */
6806         n_instructions = instr_pattern_dma_many_optimize(instructions,
6807                                                          instruction_data,
6808                                                          n_instructions);
6809
6810         return n_instructions;
6811 }
6812
6813 static int
6814 instruction_config(struct rte_swx_pipeline *p,
6815                    struct action *a,
6816                    const char **instructions,
6817                    uint32_t n_instructions)
6818 {
6819         struct instruction *instr = NULL;
6820         struct instruction_data *data = NULL;
6821         int err = 0;
6822         uint32_t i;
6823
6824         CHECK(n_instructions, EINVAL);
6825         CHECK(instructions, EINVAL);
6826         for (i = 0; i < n_instructions; i++)
6827                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6828
6829         /* Memory allocation. */
6830         instr = calloc(n_instructions, sizeof(struct instruction));
6831         if (!instr) {
6832                 err = -ENOMEM;
6833                 goto error;
6834         }
6835
6836         data = calloc(n_instructions, sizeof(struct instruction_data));
6837         if (!data) {
6838                 err = -ENOMEM;
6839                 goto error;
6840         }
6841
6842         for (i = 0; i < n_instructions; i++) {
6843                 char *string = strdup(instructions[i]);
6844                 if (!string) {
6845                         err = -ENOMEM;
6846                         goto error;
6847                 }
6848
6849                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6850                 if (err) {
6851                         free(string);
6852                         goto error;
6853                 }
6854
6855                 free(string);
6856         }
6857
6858         err = instr_label_check(data, n_instructions);
6859         if (err)
6860                 goto error;
6861
6862         err = instr_verify(p, a, instr, data, n_instructions);
6863         if (err)
6864                 goto error;
6865
6866         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6867
6868         err = instr_jmp_resolve(instr, data, n_instructions);
6869         if (err)
6870                 goto error;
6871
6872         if (a) {
6873                 a->instructions = instr;
6874                 a->instruction_data = data;
6875                 a->n_instructions = n_instructions;
6876         } else {
6877                 p->instructions = instr;
6878                 p->instruction_data = data;
6879                 p->n_instructions = n_instructions;
6880         }
6881
6882         return 0;
6883
6884 error:
6885         free(data);
6886         free(instr);
6887         return err;
6888 }
6889
6890 static instr_exec_t instruction_table[] = {
6891         [INSTR_RX] = instr_rx_exec,
6892         [INSTR_TX] = instr_tx_exec,
6893         [INSTR_TX_I] = instr_tx_i_exec,
6894         [INSTR_DROP] = instr_drop_exec,
6895         [INSTR_MIRROR] = instr_mirror_exec,
6896         [INSTR_RECIRCULATE] = instr_recirculate_exec,
6897         [INSTR_RECIRCID] = instr_recircid_exec,
6898
6899         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6900         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6901         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6902         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6903         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6904         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6905         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6906         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6907         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6908         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6909
6910         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6911         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6912         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6913         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6914         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6915         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6916         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6917         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6918         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6919
6920         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6921         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6922
6923         [INSTR_MOV] = instr_mov_exec,
6924         [INSTR_MOV_MH] = instr_mov_mh_exec,
6925         [INSTR_MOV_HM] = instr_mov_hm_exec,
6926         [INSTR_MOV_HH] = instr_mov_hh_exec,
6927         [INSTR_MOV_I] = instr_mov_i_exec,
6928
6929         [INSTR_DMA_HT] = instr_dma_ht_exec,
6930         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6931         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6932         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6933         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6934         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6935         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6936         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6937
6938         [INSTR_ALU_ADD] = instr_alu_add_exec,
6939         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6940         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6941         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6942         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6943         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6944
6945         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6946         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6947         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6948         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6949         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6950         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6951
6952         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6953         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6954         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6955         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6956
6957         [INSTR_ALU_AND] = instr_alu_and_exec,
6958         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6959         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6960         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6961         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6962
6963         [INSTR_ALU_OR] = instr_alu_or_exec,
6964         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6965         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6966         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6967         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6968
6969         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6970         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6971         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6972         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6973         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6974
6975         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6976         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6977         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6978         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6979         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6980         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6981
6982         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6983         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6984         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6985         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6986         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6987         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6988
6989         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6990         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6991         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6992
6993         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6994         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6995         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6996         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6997         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6998         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6999
7000         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7001         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7002         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7003         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7004         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7005         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7006         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7007         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7008         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7009
7010         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7011         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7012         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7013         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7014         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7015         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7016         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7017         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7018         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7019
7020         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7021         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7022         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7023
7024         [INSTR_METER_HHM] = instr_meter_hhm_exec,
7025         [INSTR_METER_HHI] = instr_meter_hhi_exec,
7026         [INSTR_METER_HMM] = instr_meter_hmm_exec,
7027         [INSTR_METER_HMI] = instr_meter_hmi_exec,
7028         [INSTR_METER_MHM] = instr_meter_mhm_exec,
7029         [INSTR_METER_MHI] = instr_meter_mhi_exec,
7030         [INSTR_METER_MMM] = instr_meter_mmm_exec,
7031         [INSTR_METER_MMI] = instr_meter_mmi_exec,
7032         [INSTR_METER_IHM] = instr_meter_ihm_exec,
7033         [INSTR_METER_IHI] = instr_meter_ihi_exec,
7034         [INSTR_METER_IMM] = instr_meter_imm_exec,
7035         [INSTR_METER_IMI] = instr_meter_imi_exec,
7036
7037         [INSTR_TABLE] = instr_table_exec,
7038         [INSTR_TABLE_AF] = instr_table_af_exec,
7039         [INSTR_SELECTOR] = instr_selector_exec,
7040         [INSTR_LEARNER] = instr_learner_exec,
7041         [INSTR_LEARNER_AF] = instr_learner_af_exec,
7042         [INSTR_LEARNER_LEARN] = instr_learn_exec,
7043         [INSTR_LEARNER_FORGET] = instr_forget_exec,
7044         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7045         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7046
7047         [INSTR_JMP] = instr_jmp_exec,
7048         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7049         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7050         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7051         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7052         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7053         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7054
7055         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7056         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7057         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7058         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7059         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7060
7061         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7062         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7063         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7064         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7065         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7066
7067         [INSTR_JMP_LT] = instr_jmp_lt_exec,
7068         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7069         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7070         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7071         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7072         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7073
7074         [INSTR_JMP_GT] = instr_jmp_gt_exec,
7075         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7076         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7077         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7078         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7079         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7080
7081         [INSTR_RETURN] = instr_return_exec,
7082 };
7083
7084 static int
7085 instruction_table_build(struct rte_swx_pipeline *p)
7086 {
7087         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
7088                                       sizeof(struct instr_exec_t *));
7089         if (!p->instruction_table)
7090                 return -EINVAL;
7091
7092         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
7093
7094         return 0;
7095 }
7096
7097 static void
7098 instruction_table_build_free(struct rte_swx_pipeline *p)
7099 {
7100         if (!p->instruction_table)
7101                 return;
7102
7103         free(p->instruction_table);
7104         p->instruction_table = NULL;
7105 }
7106
7107 static void
7108 instruction_table_free(struct rte_swx_pipeline *p)
7109 {
7110         instruction_table_build_free(p);
7111 }
7112
7113 static inline void
7114 instr_exec(struct rte_swx_pipeline *p)
7115 {
7116         struct thread *t = &p->threads[p->thread_id];
7117         struct instruction *ip = t->ip;
7118         instr_exec_t instr = p->instruction_table[ip->type];
7119
7120         instr(p);
7121 }
7122
7123 /*
7124  * Action.
7125  */
7126 static struct action *
7127 action_find(struct rte_swx_pipeline *p, const char *name)
7128 {
7129         struct action *elem;
7130
7131         if (!name)
7132                 return NULL;
7133
7134         TAILQ_FOREACH(elem, &p->actions, node)
7135                 if (strcmp(elem->name, name) == 0)
7136                         return elem;
7137
7138         return NULL;
7139 }
7140
7141 static struct action *
7142 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7143 {
7144         struct action *action = NULL;
7145
7146         TAILQ_FOREACH(action, &p->actions, node)
7147                 if (action->id == id)
7148                         return action;
7149
7150         return NULL;
7151 }
7152
7153 static struct field *
7154 action_field_find(struct action *a, const char *name)
7155 {
7156         return a->st ? struct_type_field_find(a->st, name) : NULL;
7157 }
7158
7159 static struct field *
7160 action_field_parse(struct action *action, const char *name)
7161 {
7162         if (name[0] != 't' || name[1] != '.')
7163                 return NULL;
7164
7165         return action_field_find(action, &name[2]);
7166 }
7167
7168 static int
7169 action_has_nbo_args(struct action *a)
7170 {
7171         uint32_t i;
7172
7173         /* Return if the action does not have any args. */
7174         if (!a->st)
7175                 return 0; /* FALSE */
7176
7177         for (i = 0; i < a->st->n_fields; i++)
7178                 if (a->args_endianness[i])
7179                         return 1; /* TRUE */
7180
7181         return 0; /* FALSE */
7182 }
7183
7184 static int
7185 action_does_learning(struct action *a)
7186 {
7187         uint32_t i;
7188
7189         for (i = 0; i < a->n_instructions; i++)
7190                 switch (a->instructions[i].type) {
7191                 case INSTR_LEARNER_LEARN:
7192                         return 1; /* TRUE */
7193
7194                 case INSTR_LEARNER_FORGET:
7195                         return 1; /* TRUE */
7196
7197                 default:
7198                         continue;
7199                 }
7200
7201         return 0; /* FALSE */
7202 }
7203
7204 int
7205 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7206                                const char *name,
7207                                const char *args_struct_type_name,
7208                                const char **instructions,
7209                                uint32_t n_instructions)
7210 {
7211         struct struct_type *args_struct_type = NULL;
7212         struct action *a = NULL;
7213         int status = 0;
7214
7215         CHECK(p, EINVAL);
7216
7217         CHECK_NAME(name, EINVAL);
7218         CHECK(!action_find(p, name), EEXIST);
7219
7220         if (args_struct_type_name) {
7221                 CHECK_NAME(args_struct_type_name, EINVAL);
7222                 args_struct_type = struct_type_find(p, args_struct_type_name);
7223                 CHECK(args_struct_type, EINVAL);
7224                 CHECK(!args_struct_type->var_size, EINVAL);
7225         }
7226
7227         /* Node allocation. */
7228         a = calloc(1, sizeof(struct action));
7229         if (!a) {
7230                 status = -ENOMEM;
7231                 goto error;
7232         }
7233
7234         if (args_struct_type) {
7235                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7236                 if (!a->args_endianness) {
7237                         status = -ENOMEM;
7238                         goto error;
7239                 }
7240         }
7241
7242         /* Node initialization. */
7243         strcpy(a->name, name);
7244         a->st = args_struct_type;
7245         a->id = p->n_actions;
7246
7247         /* Instruction translation. */
7248         status = instruction_config(p, a, instructions, n_instructions);
7249         if (status)
7250                 goto error;
7251
7252         /* Node add to tailq. */
7253         TAILQ_INSERT_TAIL(&p->actions, a, node);
7254         p->n_actions++;
7255
7256         return 0;
7257
7258 error:
7259         if (!a)
7260                 return status;
7261
7262         free(a->args_endianness);
7263         free(a->instructions);
7264         free(a->instruction_data);
7265         free(a);
7266
7267         return status;
7268 }
7269
7270 static int
7271 action_build(struct rte_swx_pipeline *p)
7272 {
7273         struct action *action;
7274
7275         /* p->action_instructions. */
7276         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7277         CHECK(p->action_instructions, ENOMEM);
7278
7279         TAILQ_FOREACH(action, &p->actions, node)
7280                 p->action_instructions[action->id] = action->instructions;
7281
7282         /* p->action_funcs. */
7283         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7284         CHECK(p->action_funcs, ENOMEM);
7285
7286         return 0;
7287 }
7288
7289 static void
7290 action_build_free(struct rte_swx_pipeline *p)
7291 {
7292         free(p->action_funcs);
7293         p->action_funcs = NULL;
7294
7295         free(p->action_instructions);
7296         p->action_instructions = NULL;
7297 }
7298
7299 static void
7300 action_free(struct rte_swx_pipeline *p)
7301 {
7302         action_build_free(p);
7303
7304         for ( ; ; ) {
7305                 struct action *action;
7306
7307                 action = TAILQ_FIRST(&p->actions);
7308                 if (!action)
7309                         break;
7310
7311                 TAILQ_REMOVE(&p->actions, action, node);
7312                 free(action->args_endianness);
7313                 free(action->instructions);
7314                 free(action->instruction_data);
7315                 free(action);
7316         }
7317 }
7318
7319 static uint32_t
7320 action_arg_src_mov_count(struct action *a,
7321                          uint32_t arg_id,
7322                          struct instruction *instructions,
7323                          struct instruction_data *instruction_data,
7324                          uint32_t n_instructions)
7325 {
7326         uint32_t offset, n_users = 0, i;
7327
7328         if (!a->st ||
7329             (arg_id >= a->st->n_fields) ||
7330             !instructions ||
7331             !instruction_data ||
7332             !n_instructions)
7333                 return 0;
7334
7335         offset = a->st->fields[arg_id].offset / 8;
7336
7337         for (i = 0; i < n_instructions; i++) {
7338                 struct instruction *instr = &instructions[i];
7339                 struct instruction_data *data = &instruction_data[i];
7340
7341                 if (data->invalid ||
7342                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7343                     instr->mov.src.struct_id ||
7344                     (instr->mov.src.offset != offset))
7345                         continue;
7346
7347                 n_users++;
7348         }
7349
7350         return n_users;
7351 }
7352
7353 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7354 #define field_ntoh(val, n_bits) (ntoh64((val) << (64 - n_bits)))
7355 #define field_hton(val, n_bits) (hton64((val) << (64 - n_bits)))
7356 #else
7357 #define field_ntoh(val, n_bits) (val)
7358 #define field_hton(val, n_bits) (val)
7359 #endif
7360
7361 #define ACTION_ARGS_TOKENS_MAX 256
7362
7363 static int
7364 action_args_parse(struct action *a, const char *args, uint8_t *data)
7365 {
7366         char *tokens[ACTION_ARGS_TOKENS_MAX], *s0 = NULL, *s;
7367         uint32_t n_tokens = 0, offset = 0, i;
7368         int status = 0;
7369
7370         /* Checks. */
7371         if (!a->st || !args || !args[0]) {
7372                 status = -EINVAL;
7373                 goto error;
7374         }
7375
7376         /* Memory allocation. */
7377         s0 = strdup(args);
7378         if (!s0) {
7379                 status = -ENOMEM;
7380                 goto error;
7381         }
7382
7383         /* Parse the string into tokens. */
7384         for (s = s0; ; ) {
7385                 char *token;
7386
7387                 token = strtok_r(s, " \f\n\r\t\v", &s);
7388                 if (!token)
7389                         break;
7390
7391                 if (n_tokens >= RTE_DIM(tokens)) {
7392                         status = -EINVAL;
7393                         goto error;
7394                 }
7395
7396                 tokens[n_tokens] = token;
7397                 n_tokens++;
7398         }
7399
7400         /* More checks. */
7401         if (n_tokens != a->st->n_fields * 2) {
7402                 status = -EINVAL;
7403                 goto error;
7404         }
7405
7406         /* Process the action arguments. */
7407         for (i = 0; i < a->st->n_fields; i++) {
7408                 struct field *f = &a->st->fields[i];
7409                 char *arg_name = tokens[i * 2];
7410                 char *arg_val = tokens[i * 2 + 1];
7411                 uint64_t val;
7412
7413                 if (strcmp(arg_name, f->name)) {
7414                         status = -EINVAL;
7415                         goto error;
7416                 }
7417
7418                 val = strtoull(arg_val, &arg_val, 0);
7419                 if (arg_val[0]) {
7420                         status = -EINVAL;
7421                         goto error;
7422                 }
7423
7424                 /* Endianness conversion. */
7425                 if (a->args_endianness[i])
7426                         val = field_hton(val, f->n_bits);
7427
7428                 /* Copy to entry. */
7429                 memcpy(&data[offset], (uint8_t *)&val, f->n_bits / 8);
7430                 offset += f->n_bits / 8;
7431         }
7432
7433 error:
7434         free(s0);
7435         return status;
7436 }
7437
7438 /*
7439  * Table.
7440  */
7441 static struct table_type *
7442 table_type_find(struct rte_swx_pipeline *p, const char *name)
7443 {
7444         struct table_type *elem;
7445
7446         TAILQ_FOREACH(elem, &p->table_types, node)
7447                 if (strcmp(elem->name, name) == 0)
7448                         return elem;
7449
7450         return NULL;
7451 }
7452
7453 static struct table_type *
7454 table_type_resolve(struct rte_swx_pipeline *p,
7455                    const char *recommended_type_name,
7456                    enum rte_swx_table_match_type match_type)
7457 {
7458         struct table_type *elem;
7459
7460         /* Only consider the recommended type if the match type is correct. */
7461         if (recommended_type_name)
7462                 TAILQ_FOREACH(elem, &p->table_types, node)
7463                         if (!strcmp(elem->name, recommended_type_name) &&
7464                             (elem->match_type == match_type))
7465                                 return elem;
7466
7467         /* Ignore the recommended type and get the first element with this match
7468          * type.
7469          */
7470         TAILQ_FOREACH(elem, &p->table_types, node)
7471                 if (elem->match_type == match_type)
7472                         return elem;
7473
7474         return NULL;
7475 }
7476
7477 static struct table *
7478 table_find(struct rte_swx_pipeline *p, const char *name)
7479 {
7480         struct table *elem;
7481
7482         TAILQ_FOREACH(elem, &p->tables, node)
7483                 if (strcmp(elem->name, name) == 0)
7484                         return elem;
7485
7486         return NULL;
7487 }
7488
7489 static struct table *
7490 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7491 {
7492         struct table *table = NULL;
7493
7494         TAILQ_FOREACH(table, &p->tables, node)
7495                 if (table->id == id)
7496                         return table;
7497
7498         return NULL;
7499 }
7500
7501 int
7502 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7503                                      const char *name,
7504                                      enum rte_swx_table_match_type match_type,
7505                                      struct rte_swx_table_ops *ops)
7506 {
7507         struct table_type *elem;
7508
7509         CHECK(p, EINVAL);
7510
7511         CHECK_NAME(name, EINVAL);
7512         CHECK(!table_type_find(p, name), EEXIST);
7513
7514         CHECK(ops, EINVAL);
7515         CHECK(ops->create, EINVAL);
7516         CHECK(ops->lkp, EINVAL);
7517         CHECK(ops->free, EINVAL);
7518
7519         /* Node allocation. */
7520         elem = calloc(1, sizeof(struct table_type));
7521         CHECK(elem, ENOMEM);
7522
7523         /* Node initialization. */
7524         strcpy(elem->name, name);
7525         elem->match_type = match_type;
7526         memcpy(&elem->ops, ops, sizeof(*ops));
7527
7528         /* Node add to tailq. */
7529         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7530
7531         return 0;
7532 }
7533
7534 static int
7535 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7536                          uint32_t n_fields,
7537                          enum rte_swx_table_match_type *match_type)
7538 {
7539         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7540
7541         for (i = 0; i < n_fields; i++) {
7542                 struct rte_swx_match_field_params  *f = &fields[i];
7543
7544                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7545                         n_fields_em++;
7546
7547                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7548                         n_fields_lpm++;
7549         }
7550
7551         if ((n_fields_lpm > 1) ||
7552             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7553                 return -EINVAL;
7554
7555         *match_type = (n_fields_em == n_fields) ?
7556                        RTE_SWX_TABLE_MATCH_EXACT :
7557                        RTE_SWX_TABLE_MATCH_WILDCARD;
7558
7559         return 0;
7560 }
7561
7562 static int
7563 table_match_fields_check(struct rte_swx_pipeline *p,
7564                          struct rte_swx_pipeline_table_params *params,
7565                          struct header **header)
7566 {
7567         struct header *h0 = NULL;
7568         struct field *hf, *mf;
7569         uint32_t *offset = NULL, i;
7570         int status = 0;
7571
7572         /* Return if no match fields. */
7573         if (!params->n_fields) {
7574                 if (params->fields) {
7575                         status = -EINVAL;
7576                         goto end;
7577                 }
7578
7579                 if (header)
7580                         *header = NULL;
7581
7582                 return 0;
7583         }
7584
7585         /* Memory allocation. */
7586         offset = calloc(params->n_fields, sizeof(uint32_t));
7587         if (!offset) {
7588                 status = -ENOMEM;
7589                 goto end;
7590         }
7591
7592         /* Check that all the match fields belong to either the same header or
7593          * to the meta-data.
7594          */
7595         hf = header_field_parse(p, params->fields[0].name, &h0);
7596         mf = metadata_field_parse(p, params->fields[0].name);
7597         if ((!hf && !mf) || (hf && hf->var_size)) {
7598                 status = -EINVAL;
7599                 goto end;
7600         }
7601
7602         offset[0] = h0 ? hf->offset : mf->offset;
7603
7604         for (i = 1; i < params->n_fields; i++)
7605                 if (h0) {
7606                         struct header *h;
7607
7608                         hf = header_field_parse(p, params->fields[i].name, &h);
7609                         if (!hf || (h->id != h0->id) || hf->var_size) {
7610                                 status = -EINVAL;
7611                                 goto end;
7612                         }
7613
7614                         offset[i] = hf->offset;
7615                 } else {
7616                         mf = metadata_field_parse(p, params->fields[i].name);
7617                         if (!mf) {
7618                                 status = -EINVAL;
7619                                 goto end;
7620                         }
7621
7622                         offset[i] = mf->offset;
7623                 }
7624
7625         /* Check that there are no duplicated match fields. */
7626         for (i = 0; i < params->n_fields; i++) {
7627                 uint32_t j;
7628
7629                 for (j = 0; j < i; j++)
7630                         if (offset[j] == offset[i]) {
7631                                 status = -EINVAL;
7632                                 goto end;
7633                         }
7634         }
7635
7636         /* Return. */
7637         if (header)
7638                 *header = h0;
7639
7640 end:
7641         free(offset);
7642         return status;
7643 }
7644
7645 int
7646 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7647                               const char *name,
7648                               struct rte_swx_pipeline_table_params *params,
7649                               const char *recommended_table_type_name,
7650                               const char *args,
7651                               uint32_t size)
7652 {
7653         struct table_type *type;
7654         struct table *t = NULL;
7655         struct action *default_action;
7656         struct header *header = NULL;
7657         uint32_t action_data_size_max = 0, i;
7658         int status = 0;
7659
7660         CHECK(p, EINVAL);
7661
7662         CHECK_NAME(name, EINVAL);
7663         CHECK(!table_find(p, name), EEXIST);
7664         CHECK(!selector_find(p, name), EEXIST);
7665         CHECK(!learner_find(p, name), EEXIST);
7666
7667         CHECK(params, EINVAL);
7668
7669         /* Match checks. */
7670         status = table_match_fields_check(p, params, &header);
7671         if (status)
7672                 return status;
7673
7674         /* Action checks. */
7675         CHECK(params->n_actions, EINVAL);
7676         CHECK(params->action_names, EINVAL);
7677         for (i = 0; i < params->n_actions; i++) {
7678                 const char *action_name = params->action_names[i];
7679                 struct action *a;
7680                 uint32_t action_data_size;
7681                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7682
7683                 CHECK_NAME(action_name, EINVAL);
7684
7685                 a = action_find(p, action_name);
7686                 CHECK(a, EINVAL);
7687                 CHECK(!action_does_learning(a), EINVAL);
7688
7689                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7690                 if (action_data_size > action_data_size_max)
7691                         action_data_size_max = action_data_size;
7692
7693                 if (params->action_is_for_table_entries)
7694                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7695                 if (params->action_is_for_default_entry)
7696                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7697                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7698         }
7699
7700         CHECK_NAME(params->default_action_name, EINVAL);
7701         for (i = 0; i < p->n_actions; i++)
7702                 if (!strcmp(params->action_names[i],
7703                             params->default_action_name))
7704                         break;
7705         CHECK(i < params->n_actions, EINVAL);
7706         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7707               EINVAL);
7708
7709         default_action = action_find(p, params->default_action_name);
7710         CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
7711               EINVAL);
7712
7713         /* Table type checks. */
7714         if (recommended_table_type_name)
7715                 CHECK_NAME(recommended_table_type_name, EINVAL);
7716
7717         if (params->n_fields) {
7718                 enum rte_swx_table_match_type match_type;
7719
7720                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7721                 if (status)
7722                         return status;
7723
7724                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7725                 CHECK(type, EINVAL);
7726         } else {
7727                 type = NULL;
7728         }
7729
7730         /* Memory allocation. */
7731         t = calloc(1, sizeof(struct table));
7732         if (!t) {
7733                 status = -ENOMEM;
7734                 goto error;
7735         }
7736
7737         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7738         if (!t->fields) {
7739                 status = -ENOMEM;
7740                 goto error;
7741         }
7742
7743         t->actions = calloc(params->n_actions, sizeof(struct action *));
7744         if (!t->actions) {
7745                 status = -ENOMEM;
7746                 goto error;
7747         }
7748
7749         if (action_data_size_max) {
7750                 t->default_action_data = calloc(1, action_data_size_max);
7751                 if (!t->default_action_data) {
7752                         status = -ENOMEM;
7753                         goto error;
7754                 }
7755         }
7756
7757         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7758         if (!t->action_is_for_table_entries) {
7759                 status = -ENOMEM;
7760                 goto error;
7761         }
7762
7763         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7764         if (!t->action_is_for_default_entry) {
7765                 status = -ENOMEM;
7766                 goto error;
7767         }
7768
7769         /* Node initialization. */
7770         strcpy(t->name, name);
7771         if (args && args[0])
7772                 strcpy(t->args, args);
7773         t->type = type;
7774
7775         for (i = 0; i < params->n_fields; i++) {
7776                 struct rte_swx_match_field_params *field = &params->fields[i];
7777                 struct match_field *f = &t->fields[i];
7778
7779                 f->match_type = field->match_type;
7780                 f->field = header ?
7781                         header_field_parse(p, field->name, NULL) :
7782                         metadata_field_parse(p, field->name);
7783         }
7784         t->n_fields = params->n_fields;
7785         t->header = header;
7786
7787         for (i = 0; i < params->n_actions; i++) {
7788                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7789
7790                 if (params->action_is_for_table_entries)
7791                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7792                 if (params->action_is_for_default_entry)
7793                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7794
7795                 t->actions[i] = action_find(p, params->action_names[i]);
7796                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7797                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7798         }
7799         t->default_action = default_action;
7800         if (default_action->st) {
7801                 status = action_args_parse(default_action,
7802                                            params->default_action_args,
7803                                            t->default_action_data);
7804                 if (status)
7805                         goto error;
7806         }
7807
7808         t->n_actions = params->n_actions;
7809         t->default_action_is_const = params->default_action_is_const;
7810         t->action_data_size_max = action_data_size_max;
7811
7812         t->size = size;
7813         t->id = p->n_tables;
7814
7815         /* Node add to tailq. */
7816         TAILQ_INSERT_TAIL(&p->tables, t, node);
7817         p->n_tables++;
7818
7819         return 0;
7820
7821 error:
7822         if (!t)
7823                 return status;
7824
7825         free(t->action_is_for_default_entry);
7826         free(t->action_is_for_table_entries);
7827         free(t->default_action_data);
7828         free(t->actions);
7829         free(t->fields);
7830         free(t);
7831
7832         return status;
7833 }
7834
7835 static struct rte_swx_table_params *
7836 table_params_get(struct table *table)
7837 {
7838         struct rte_swx_table_params *params;
7839         struct field *first, *last;
7840         uint8_t *key_mask;
7841         uint32_t key_size, key_offset, action_data_size, i;
7842
7843         /* Memory allocation. */
7844         params = calloc(1, sizeof(struct rte_swx_table_params));
7845         if (!params)
7846                 return NULL;
7847
7848         /* Find first (smallest offset) and last (biggest offset) match fields. */
7849         first = table->fields[0].field;
7850         last = table->fields[0].field;
7851
7852         for (i = 0; i < table->n_fields; i++) {
7853                 struct field *f = table->fields[i].field;
7854
7855                 if (f->offset < first->offset)
7856                         first = f;
7857
7858                 if (f->offset > last->offset)
7859                         last = f;
7860         }
7861
7862         /* Key offset and size. */
7863         key_offset = first->offset / 8;
7864         key_size = (last->offset + last->n_bits - first->offset) / 8;
7865
7866         /* Memory allocation. */
7867         key_mask = calloc(1, key_size);
7868         if (!key_mask) {
7869                 free(params);
7870                 return NULL;
7871         }
7872
7873         /* Key mask. */
7874         for (i = 0; i < table->n_fields; i++) {
7875                 struct field *f = table->fields[i].field;
7876                 uint32_t start = (f->offset - first->offset) / 8;
7877                 size_t size = f->n_bits / 8;
7878
7879                 memset(&key_mask[start], 0xFF, size);
7880         }
7881
7882         /* Action data size. */
7883         action_data_size = 0;
7884         for (i = 0; i < table->n_actions; i++) {
7885                 struct action *action = table->actions[i];
7886                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7887
7888                 if (ads > action_data_size)
7889                         action_data_size = ads;
7890         }
7891
7892         /* Fill in. */
7893         params->match_type = table->type->match_type;
7894         params->key_size = key_size;
7895         params->key_offset = key_offset;
7896         params->key_mask0 = key_mask;
7897         params->action_data_size = action_data_size;
7898         params->n_keys_max = table->size;
7899
7900         return params;
7901 }
7902
7903 static void
7904 table_params_free(struct rte_swx_table_params *params)
7905 {
7906         if (!params)
7907                 return;
7908
7909         free(params->key_mask0);
7910         free(params);
7911 }
7912
7913 static int
7914 table_stub_lkp(void *table __rte_unused,
7915                void *mailbox __rte_unused,
7916                uint8_t **key __rte_unused,
7917                uint64_t *action_id __rte_unused,
7918                uint8_t **action_data __rte_unused,
7919                int *hit)
7920 {
7921         *hit = 0;
7922         return 1; /* DONE. */
7923 }
7924
7925 static int
7926 table_build(struct rte_swx_pipeline *p)
7927 {
7928         uint32_t i;
7929
7930         /* Per pipeline: table statistics. */
7931         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7932         CHECK(p->table_stats, ENOMEM);
7933
7934         for (i = 0; i < p->n_tables; i++) {
7935                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7936                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7937         }
7938
7939         /* Per thread: table runt-time. */
7940         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7941                 struct thread *t = &p->threads[i];
7942                 struct table *table;
7943
7944                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7945                 CHECK(t->tables, ENOMEM);
7946
7947                 TAILQ_FOREACH(table, &p->tables, node) {
7948                         struct table_runtime *r = &t->tables[table->id];
7949
7950                         if (table->type) {
7951                                 uint64_t size;
7952
7953                                 size = table->type->ops.mailbox_size_get();
7954
7955                                 /* r->func. */
7956                                 r->func = table->type->ops.lkp;
7957
7958                                 /* r->mailbox. */
7959                                 if (size) {
7960                                         r->mailbox = calloc(1, size);
7961                                         CHECK(r->mailbox, ENOMEM);
7962                                 }
7963
7964                                 /* r->key. */
7965                                 r->key = table->header ?
7966                                         &t->structs[table->header->struct_id] :
7967                                         &t->structs[p->metadata_struct_id];
7968                         } else {
7969                                 r->func = table_stub_lkp;
7970                         }
7971                 }
7972         }
7973
7974         return 0;
7975 }
7976
7977 static void
7978 table_build_free(struct rte_swx_pipeline *p)
7979 {
7980         uint32_t i;
7981
7982         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7983                 struct thread *t = &p->threads[i];
7984                 uint32_t j;
7985
7986                 if (!t->tables)
7987                         continue;
7988
7989                 for (j = 0; j < p->n_tables; j++) {
7990                         struct table_runtime *r = &t->tables[j];
7991
7992                         free(r->mailbox);
7993                 }
7994
7995                 free(t->tables);
7996                 t->tables = NULL;
7997         }
7998
7999         if (p->table_stats) {
8000                 for (i = 0; i < p->n_tables; i++)
8001                         free(p->table_stats[i].n_pkts_action);
8002
8003                 free(p->table_stats);
8004         }
8005 }
8006
8007 static void
8008 table_free(struct rte_swx_pipeline *p)
8009 {
8010         table_build_free(p);
8011
8012         /* Tables. */
8013         for ( ; ; ) {
8014                 struct table *elem;
8015
8016                 elem = TAILQ_FIRST(&p->tables);
8017                 if (!elem)
8018                         break;
8019
8020                 TAILQ_REMOVE(&p->tables, elem, node);
8021                 free(elem->fields);
8022                 free(elem->actions);
8023                 free(elem->default_action_data);
8024                 free(elem);
8025         }
8026
8027         /* Table types. */
8028         for ( ; ; ) {
8029                 struct table_type *elem;
8030
8031                 elem = TAILQ_FIRST(&p->table_types);
8032                 if (!elem)
8033                         break;
8034
8035                 TAILQ_REMOVE(&p->table_types, elem, node);
8036                 free(elem);
8037         }
8038 }
8039
8040 /*
8041  * Selector.
8042  */
8043 static struct selector *
8044 selector_find(struct rte_swx_pipeline *p, const char *name)
8045 {
8046         struct selector *s;
8047
8048         TAILQ_FOREACH(s, &p->selectors, node)
8049                 if (strcmp(s->name, name) == 0)
8050                         return s;
8051
8052         return NULL;
8053 }
8054
8055 static struct selector *
8056 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8057 {
8058         struct selector *s = NULL;
8059
8060         TAILQ_FOREACH(s, &p->selectors, node)
8061                 if (s->id == id)
8062                         return s;
8063
8064         return NULL;
8065 }
8066
8067 static int
8068 selector_fields_check(struct rte_swx_pipeline *p,
8069                       struct rte_swx_pipeline_selector_params *params,
8070                       struct header **header)
8071 {
8072         struct header *h0 = NULL;
8073         struct field *hf, *mf;
8074         uint32_t i;
8075
8076         /* Return if no selector fields. */
8077         if (!params->n_selector_fields || !params->selector_field_names)
8078                 return -EINVAL;
8079
8080         /* Check that all the selector fields either belong to the same header
8081          * or are all meta-data fields.
8082          */
8083         hf = header_field_parse(p, params->selector_field_names[0], &h0);
8084         mf = metadata_field_parse(p, params->selector_field_names[0]);
8085         if (!hf && !mf)
8086                 return -EINVAL;
8087
8088         for (i = 1; i < params->n_selector_fields; i++)
8089                 if (h0) {
8090                         struct header *h;
8091
8092                         hf = header_field_parse(p, params->selector_field_names[i], &h);
8093                         if (!hf || (h->id != h0->id))
8094                                 return -EINVAL;
8095                 } else {
8096                         mf = metadata_field_parse(p, params->selector_field_names[i]);
8097                         if (!mf)
8098                                 return -EINVAL;
8099                 }
8100
8101         /* Check that there are no duplicated match fields. */
8102         for (i = 0; i < params->n_selector_fields; i++) {
8103                 const char *field_name = params->selector_field_names[i];
8104                 uint32_t j;
8105
8106                 for (j = i + 1; j < params->n_selector_fields; j++)
8107                         if (!strcmp(params->selector_field_names[j], field_name))
8108                                 return -EINVAL;
8109         }
8110
8111         /* Return. */
8112         if (header)
8113                 *header = h0;
8114
8115         return 0;
8116 }
8117
8118 int
8119 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8120                                  const char *name,
8121                                  struct rte_swx_pipeline_selector_params *params)
8122 {
8123         struct selector *s;
8124         struct header *selector_header = NULL;
8125         struct field *group_id_field, *member_id_field;
8126         uint32_t i;
8127         int status = 0;
8128
8129         CHECK(p, EINVAL);
8130
8131         CHECK_NAME(name, EINVAL);
8132         CHECK(!table_find(p, name), EEXIST);
8133         CHECK(!selector_find(p, name), EEXIST);
8134         CHECK(!learner_find(p, name), EEXIST);
8135
8136         CHECK(params, EINVAL);
8137
8138         CHECK_NAME(params->group_id_field_name, EINVAL);
8139         group_id_field = metadata_field_parse(p, params->group_id_field_name);
8140         CHECK(group_id_field, EINVAL);
8141
8142         for (i = 0; i < params->n_selector_fields; i++) {
8143                 const char *field_name = params->selector_field_names[i];
8144
8145                 CHECK_NAME(field_name, EINVAL);
8146         }
8147         status = selector_fields_check(p, params, &selector_header);
8148         if (status)
8149                 return status;
8150
8151         CHECK_NAME(params->member_id_field_name, EINVAL);
8152         member_id_field = metadata_field_parse(p, params->member_id_field_name);
8153         CHECK(member_id_field, EINVAL);
8154
8155         CHECK(params->n_groups_max, EINVAL);
8156
8157         CHECK(params->n_members_per_group_max, EINVAL);
8158
8159         /* Memory allocation. */
8160         s = calloc(1, sizeof(struct selector));
8161         if (!s) {
8162                 status = -ENOMEM;
8163                 goto error;
8164         }
8165
8166         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8167         if (!s->selector_fields) {
8168                 status = -ENOMEM;
8169                 goto error;
8170         }
8171
8172         /* Node initialization. */
8173         strcpy(s->name, name);
8174
8175         s->group_id_field = group_id_field;
8176
8177         for (i = 0; i < params->n_selector_fields; i++) {
8178                 const char *field_name = params->selector_field_names[i];
8179
8180                 s->selector_fields[i] = selector_header ?
8181                         header_field_parse(p, field_name, NULL) :
8182                         metadata_field_parse(p, field_name);
8183         }
8184
8185         s->n_selector_fields = params->n_selector_fields;
8186
8187         s->selector_header = selector_header;
8188
8189         s->member_id_field = member_id_field;
8190
8191         s->n_groups_max = params->n_groups_max;
8192
8193         s->n_members_per_group_max = params->n_members_per_group_max;
8194
8195         s->id = p->n_selectors;
8196
8197         /* Node add to tailq. */
8198         TAILQ_INSERT_TAIL(&p->selectors, s, node);
8199         p->n_selectors++;
8200
8201         return 0;
8202
8203 error:
8204         if (!s)
8205                 return status;
8206
8207         free(s->selector_fields);
8208
8209         free(s);
8210
8211         return status;
8212 }
8213
8214 static void
8215 selector_params_free(struct rte_swx_table_selector_params *params)
8216 {
8217         if (!params)
8218                 return;
8219
8220         free(params->selector_mask);
8221
8222         free(params);
8223 }
8224
8225 static struct rte_swx_table_selector_params *
8226 selector_table_params_get(struct selector *s)
8227 {
8228         struct rte_swx_table_selector_params *params = NULL;
8229         struct field *first, *last;
8230         uint32_t i;
8231
8232         /* Memory allocation. */
8233         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8234         if (!params)
8235                 goto error;
8236
8237         /* Group ID. */
8238         params->group_id_offset = s->group_id_field->offset / 8;
8239
8240         /* Find first (smallest offset) and last (biggest offset) selector fields. */
8241         first = s->selector_fields[0];
8242         last = s->selector_fields[0];
8243
8244         for (i = 0; i < s->n_selector_fields; i++) {
8245                 struct field *f = s->selector_fields[i];
8246
8247                 if (f->offset < first->offset)
8248                         first = f;
8249
8250                 if (f->offset > last->offset)
8251                         last = f;
8252         }
8253
8254         /* Selector offset and size. */
8255         params->selector_offset = first->offset / 8;
8256         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8257
8258         /* Memory allocation. */
8259         params->selector_mask = calloc(1, params->selector_size);
8260         if (!params->selector_mask)
8261                 goto error;
8262
8263         /* Selector mask. */
8264         for (i = 0; i < s->n_selector_fields; i++) {
8265                 struct field *f = s->selector_fields[i];
8266                 uint32_t start = (f->offset - first->offset) / 8;
8267                 size_t size = f->n_bits / 8;
8268
8269                 memset(&params->selector_mask[start], 0xFF, size);
8270         }
8271
8272         /* Member ID. */
8273         params->member_id_offset = s->member_id_field->offset / 8;
8274
8275         /* Maximum number of groups. */
8276         params->n_groups_max = s->n_groups_max;
8277
8278         /* Maximum number of members per group. */
8279         params->n_members_per_group_max = s->n_members_per_group_max;
8280
8281         return params;
8282
8283 error:
8284         selector_params_free(params);
8285         return NULL;
8286 }
8287
8288 static void
8289 selector_build_free(struct rte_swx_pipeline *p)
8290 {
8291         uint32_t i;
8292
8293         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8294                 struct thread *t = &p->threads[i];
8295                 uint32_t j;
8296
8297                 if (!t->selectors)
8298                         continue;
8299
8300                 for (j = 0; j < p->n_selectors; j++) {
8301                         struct selector_runtime *r = &t->selectors[j];
8302
8303                         free(r->mailbox);
8304                 }
8305
8306                 free(t->selectors);
8307                 t->selectors = NULL;
8308         }
8309
8310         free(p->selector_stats);
8311         p->selector_stats = NULL;
8312 }
8313
8314 static int
8315 selector_build(struct rte_swx_pipeline *p)
8316 {
8317         uint32_t i;
8318         int status = 0;
8319
8320         /* Per pipeline: selector statistics. */
8321         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8322         if (!p->selector_stats) {
8323                 status = -ENOMEM;
8324                 goto error;
8325         }
8326
8327         /* Per thread: selector run-time. */
8328         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8329                 struct thread *t = &p->threads[i];
8330                 struct selector *s;
8331
8332                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8333                 if (!t->selectors) {
8334                         status = -ENOMEM;
8335                         goto error;
8336                 }
8337
8338                 TAILQ_FOREACH(s, &p->selectors, node) {
8339                         struct selector_runtime *r = &t->selectors[s->id];
8340                         uint64_t size;
8341
8342                         /* r->mailbox. */
8343                         size = rte_swx_table_selector_mailbox_size_get();
8344                         if (size) {
8345                                 r->mailbox = calloc(1, size);
8346                                 if (!r->mailbox) {
8347                                         status = -ENOMEM;
8348                                         goto error;
8349                                 }
8350                         }
8351
8352                         /* r->group_id_buffer. */
8353                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8354
8355                         /* r->selector_buffer. */
8356                         r->selector_buffer = s->selector_header ?
8357                                 &t->structs[s->selector_header->struct_id] :
8358                                 &t->structs[p->metadata_struct_id];
8359
8360                         /* r->member_id_buffer. */
8361                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8362                 }
8363         }
8364
8365         return 0;
8366
8367 error:
8368         selector_build_free(p);
8369         return status;
8370 }
8371
8372 static void
8373 selector_free(struct rte_swx_pipeline *p)
8374 {
8375         selector_build_free(p);
8376
8377         /* Selector tables. */
8378         for ( ; ; ) {
8379                 struct selector *elem;
8380
8381                 elem = TAILQ_FIRST(&p->selectors);
8382                 if (!elem)
8383                         break;
8384
8385                 TAILQ_REMOVE(&p->selectors, elem, node);
8386                 free(elem->selector_fields);
8387                 free(elem);
8388         }
8389 }
8390
8391 /*
8392  * Learner table.
8393  */
8394 static struct learner *
8395 learner_find(struct rte_swx_pipeline *p, const char *name)
8396 {
8397         struct learner *l;
8398
8399         TAILQ_FOREACH(l, &p->learners, node)
8400                 if (!strcmp(l->name, name))
8401                         return l;
8402
8403         return NULL;
8404 }
8405
8406 static struct learner *
8407 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8408 {
8409         struct learner *l = NULL;
8410
8411         TAILQ_FOREACH(l, &p->learners, node)
8412                 if (l->id == id)
8413                         return l;
8414
8415         return NULL;
8416 }
8417
8418 static int
8419 learner_match_fields_check(struct rte_swx_pipeline *p,
8420                            struct rte_swx_pipeline_learner_params *params,
8421                            struct header **header)
8422 {
8423         struct header *h0 = NULL;
8424         struct field *hf, *mf;
8425         uint32_t i;
8426
8427         /* Return if no match fields. */
8428         if (!params->n_fields || !params->field_names)
8429                 return -EINVAL;
8430
8431         /* Check that all the match fields either belong to the same header
8432          * or are all meta-data fields.
8433          */
8434         hf = header_field_parse(p, params->field_names[0], &h0);
8435         mf = metadata_field_parse(p, params->field_names[0]);
8436         if (!hf && !mf)
8437                 return -EINVAL;
8438
8439         for (i = 1; i < params->n_fields; i++)
8440                 if (h0) {
8441                         struct header *h;
8442
8443                         hf = header_field_parse(p, params->field_names[i], &h);
8444                         if (!hf || (h->id != h0->id))
8445                                 return -EINVAL;
8446                 } else {
8447                         mf = metadata_field_parse(p, params->field_names[i]);
8448                         if (!mf)
8449                                 return -EINVAL;
8450                 }
8451
8452         /* Check that there are no duplicated match fields. */
8453         for (i = 0; i < params->n_fields; i++) {
8454                 const char *field_name = params->field_names[i];
8455                 uint32_t j;
8456
8457                 for (j = i + 1; j < params->n_fields; j++)
8458                         if (!strcmp(params->field_names[j], field_name))
8459                                 return -EINVAL;
8460         }
8461
8462         /* Return. */
8463         if (header)
8464                 *header = h0;
8465
8466         return 0;
8467 }
8468
8469 static int
8470 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8471 {
8472         struct struct_type *mst = p->metadata_st, *ast = a->st;
8473         struct field *mf, *af;
8474         uint32_t mf_pos, i;
8475
8476         if (!ast) {
8477                 if (mf_name)
8478                         return -EINVAL;
8479
8480                 return 0;
8481         }
8482
8483         /* Check that mf_name is the name of a valid meta-data field. */
8484         CHECK_NAME(mf_name, EINVAL);
8485         mf = metadata_field_parse(p, mf_name);
8486         CHECK(mf, EINVAL);
8487
8488         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8489          * all the action arguments.
8490          */
8491         mf_pos = mf - mst->fields;
8492         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8493
8494         /* Check that the size of each of the identified meta-data fields matches exactly the size
8495          * of the corresponding action argument.
8496          */
8497         for (i = 0; i < ast->n_fields; i++) {
8498                 mf = &mst->fields[mf_pos + i];
8499                 af = &ast->fields[i];
8500
8501                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8502         }
8503
8504         return 0;
8505 }
8506
8507 static int
8508 learner_action_learning_check(struct rte_swx_pipeline *p,
8509                               struct action *action,
8510                               const char **action_names,
8511                               uint32_t n_actions)
8512 {
8513         uint32_t i;
8514
8515         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8516          * the action passed as argument to the "learn" instruction) is also enabled for the
8517          * current learner table.
8518          */
8519         for (i = 0; i < action->n_instructions; i++) {
8520                 struct instruction *instr = &action->instructions[i];
8521                 uint32_t found = 0, j;
8522
8523                 if (instr->type != INSTR_LEARNER_LEARN)
8524                         continue;
8525
8526                 for (j = 0; j < n_actions; j++) {
8527                         struct action *a;
8528
8529                         a = action_find(p, action_names[j]);
8530                         if (!a)
8531                                 return -EINVAL;
8532
8533                         if (a->id == instr->learn.action_id)
8534                                 found = 1;
8535                 }
8536
8537                 if (!found)
8538                         return -EINVAL;
8539         }
8540
8541         return 0;
8542 }
8543
8544 int
8545 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8546                               const char *name,
8547                               struct rte_swx_pipeline_learner_params *params,
8548                               uint32_t size,
8549                               uint32_t timeout)
8550 {
8551         struct learner *l = NULL;
8552         struct action *default_action;
8553         struct header *header = NULL;
8554         uint32_t action_data_size_max = 0, i;
8555         int status = 0;
8556
8557         CHECK(p, EINVAL);
8558
8559         CHECK_NAME(name, EINVAL);
8560         CHECK(!table_find(p, name), EEXIST);
8561         CHECK(!selector_find(p, name), EEXIST);
8562         CHECK(!learner_find(p, name), EEXIST);
8563
8564         CHECK(params, EINVAL);
8565
8566         /* Match checks. */
8567         status = learner_match_fields_check(p, params, &header);
8568         if (status)
8569                 return status;
8570
8571         /* Action checks. */
8572         CHECK(params->n_actions, EINVAL);
8573         CHECK(params->action_names, EINVAL);
8574         for (i = 0; i < params->n_actions; i++) {
8575                 const char *action_name = params->action_names[i];
8576                 struct action *a;
8577                 uint32_t action_data_size;
8578                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8579
8580                 CHECK_NAME(action_name, EINVAL);
8581
8582                 a = action_find(p, action_name);
8583                 CHECK(a, EINVAL);
8584
8585                 status = learner_action_learning_check(p,
8586                                                        a,
8587                                                        params->action_names,
8588                                                        params->n_actions);
8589                 if (status)
8590                         return status;
8591
8592                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8593                 if (action_data_size > action_data_size_max)
8594                         action_data_size_max = action_data_size;
8595
8596                 if (params->action_is_for_table_entries)
8597                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8598                 if (params->action_is_for_default_entry)
8599                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8600                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8601         }
8602
8603         CHECK_NAME(params->default_action_name, EINVAL);
8604         for (i = 0; i < p->n_actions; i++)
8605                 if (!strcmp(params->action_names[i],
8606                             params->default_action_name))
8607                         break;
8608         CHECK(i < params->n_actions, EINVAL);
8609         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8610               EINVAL);
8611
8612         default_action = action_find(p, params->default_action_name);
8613         CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
8614               EINVAL);
8615
8616         /* Any other checks. */
8617         CHECK(size, EINVAL);
8618         CHECK(timeout, EINVAL);
8619
8620         /* Memory allocation. */
8621         l = calloc(1, sizeof(struct learner));
8622         if (!l) {
8623                 status = -ENOMEM;
8624                 goto error;
8625         }
8626
8627         l->fields = calloc(params->n_fields, sizeof(struct field *));
8628         if (!l->fields) {
8629                 status = -ENOMEM;
8630                 goto error;
8631         }
8632
8633         l->actions = calloc(params->n_actions, sizeof(struct action *));
8634         if (!l->actions) {
8635                 status = -ENOMEM;
8636                 goto error;
8637         }
8638
8639         if (action_data_size_max) {
8640                 l->default_action_data = calloc(1, action_data_size_max);
8641                 if (!l->default_action_data) {
8642                         status = -ENOMEM;
8643                         goto error;
8644                 }
8645         }
8646
8647         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8648         if (!l->action_is_for_table_entries) {
8649                 status = -ENOMEM;
8650                 goto error;
8651         }
8652
8653         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8654         if (!l->action_is_for_default_entry) {
8655                 status = -ENOMEM;
8656                 goto error;
8657         }
8658
8659         /* Node initialization. */
8660         strcpy(l->name, name);
8661
8662         for (i = 0; i < params->n_fields; i++) {
8663                 const char *field_name = params->field_names[i];
8664
8665                 l->fields[i] = header ?
8666                         header_field_parse(p, field_name, NULL) :
8667                         metadata_field_parse(p, field_name);
8668         }
8669
8670         l->n_fields = params->n_fields;
8671
8672         l->header = header;
8673
8674         for (i = 0; i < params->n_actions; i++) {
8675                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8676
8677                 if (params->action_is_for_table_entries)
8678                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8679                 if (params->action_is_for_default_entry)
8680                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8681
8682                 l->actions[i] = action_find(p, params->action_names[i]);
8683                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8684                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8685         }
8686
8687         l->default_action = default_action;
8688
8689         if (default_action->st) {
8690                 status = action_args_parse(default_action,
8691                                            params->default_action_args,
8692                                            l->default_action_data);
8693                 if (status)
8694                         goto error;
8695         }
8696
8697         l->n_actions = params->n_actions;
8698
8699         l->default_action_is_const = params->default_action_is_const;
8700
8701         l->action_data_size_max = action_data_size_max;
8702
8703         l->size = size;
8704
8705         l->timeout = timeout;
8706
8707         l->id = p->n_learners;
8708
8709         /* Node add to tailq. */
8710         TAILQ_INSERT_TAIL(&p->learners, l, node);
8711         p->n_learners++;
8712
8713         return 0;
8714
8715 error:
8716         if (!l)
8717                 return status;
8718
8719         free(l->action_is_for_default_entry);
8720         free(l->action_is_for_table_entries);
8721         free(l->default_action_data);
8722         free(l->actions);
8723         free(l->fields);
8724         free(l);
8725
8726         return status;
8727 }
8728
8729 static void
8730 learner_params_free(struct rte_swx_table_learner_params *params)
8731 {
8732         if (!params)
8733                 return;
8734
8735         free(params->key_mask0);
8736
8737         free(params);
8738 }
8739
8740 static struct rte_swx_table_learner_params *
8741 learner_params_get(struct learner *l)
8742 {
8743         struct rte_swx_table_learner_params *params = NULL;
8744         struct field *first, *last;
8745         uint32_t i;
8746
8747         /* Memory allocation. */
8748         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8749         if (!params)
8750                 goto error;
8751
8752         /* Find first (smallest offset) and last (biggest offset) match fields. */
8753         first = l->fields[0];
8754         last = l->fields[0];
8755
8756         for (i = 0; i < l->n_fields; i++) {
8757                 struct field *f = l->fields[i];
8758
8759                 if (f->offset < first->offset)
8760                         first = f;
8761
8762                 if (f->offset > last->offset)
8763                         last = f;
8764         }
8765
8766         /* Key offset and size. */
8767         params->key_offset = first->offset / 8;
8768         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8769
8770         /* Memory allocation. */
8771         params->key_mask0 = calloc(1, params->key_size);
8772         if (!params->key_mask0)
8773                 goto error;
8774
8775         /* Key mask. */
8776         for (i = 0; i < l->n_fields; i++) {
8777                 struct field *f = l->fields[i];
8778                 uint32_t start = (f->offset - first->offset) / 8;
8779                 size_t size = f->n_bits / 8;
8780
8781                 memset(&params->key_mask0[start], 0xFF, size);
8782         }
8783
8784         /* Action data size. */
8785         params->action_data_size = l->action_data_size_max;
8786
8787         /* Maximum number of keys. */
8788         params->n_keys_max = l->size;
8789
8790         /* Timeout. */
8791         params->key_timeout[0] = l->timeout;
8792         params->n_key_timeouts = 1;
8793
8794         return params;
8795
8796 error:
8797         learner_params_free(params);
8798         return NULL;
8799 }
8800
8801 static void
8802 learner_build_free(struct rte_swx_pipeline *p)
8803 {
8804         uint32_t i;
8805
8806         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8807                 struct thread *t = &p->threads[i];
8808                 uint32_t j;
8809
8810                 if (!t->learners)
8811                         continue;
8812
8813                 for (j = 0; j < p->n_learners; j++) {
8814                         struct learner_runtime *r = &t->learners[j];
8815
8816                         free(r->mailbox);
8817                 }
8818
8819                 free(t->learners);
8820                 t->learners = NULL;
8821         }
8822
8823         if (p->learner_stats) {
8824                 for (i = 0; i < p->n_learners; i++)
8825                         free(p->learner_stats[i].n_pkts_action);
8826
8827                 free(p->learner_stats);
8828         }
8829 }
8830
8831 static int
8832 learner_build(struct rte_swx_pipeline *p)
8833 {
8834         uint32_t i;
8835         int status = 0;
8836
8837         /* Per pipeline: learner statistics. */
8838         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8839         CHECK(p->learner_stats, ENOMEM);
8840
8841         for (i = 0; i < p->n_learners; i++) {
8842                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8843                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8844         }
8845
8846         /* Per thread: learner run-time. */
8847         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8848                 struct thread *t = &p->threads[i];
8849                 struct learner *l;
8850
8851                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8852                 if (!t->learners) {
8853                         status = -ENOMEM;
8854                         goto error;
8855                 }
8856
8857                 TAILQ_FOREACH(l, &p->learners, node) {
8858                         struct learner_runtime *r = &t->learners[l->id];
8859                         uint64_t size;
8860
8861                         /* r->mailbox. */
8862                         size = rte_swx_table_learner_mailbox_size_get();
8863                         if (size) {
8864                                 r->mailbox = calloc(1, size);
8865                                 if (!r->mailbox) {
8866                                         status = -ENOMEM;
8867                                         goto error;
8868                                 }
8869                         }
8870
8871                         /* r->key. */
8872                         r->key = l->header ?
8873                                 &t->structs[l->header->struct_id] :
8874                                 &t->structs[p->metadata_struct_id];
8875                 }
8876         }
8877
8878         return 0;
8879
8880 error:
8881         learner_build_free(p);
8882         return status;
8883 }
8884
8885 static void
8886 learner_free(struct rte_swx_pipeline *p)
8887 {
8888         learner_build_free(p);
8889
8890         /* Learner tables. */
8891         for ( ; ; ) {
8892                 struct learner *l;
8893
8894                 l = TAILQ_FIRST(&p->learners);
8895                 if (!l)
8896                         break;
8897
8898                 TAILQ_REMOVE(&p->learners, l, node);
8899                 free(l->fields);
8900                 free(l->actions);
8901                 free(l->default_action_data);
8902                 free(l);
8903         }
8904 }
8905
8906 /*
8907  * Table state.
8908  */
8909 static int
8910 table_state_build(struct rte_swx_pipeline *p)
8911 {
8912         struct table *table;
8913         struct selector *s;
8914         struct learner *l;
8915
8916         p->table_state = calloc(p->n_tables + p->n_selectors + p->n_learners,
8917                                 sizeof(struct rte_swx_table_state));
8918         CHECK(p->table_state, ENOMEM);
8919
8920         TAILQ_FOREACH(table, &p->tables, node) {
8921                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8922
8923                 if (table->type) {
8924                         struct rte_swx_table_params *params;
8925
8926                         /* ts->obj. */
8927                         params = table_params_get(table);
8928                         CHECK(params, ENOMEM);
8929
8930                         ts->obj = table->type->ops.create(params,
8931                                 NULL,
8932                                 table->args,
8933                                 p->numa_node);
8934
8935                         table_params_free(params);
8936                         CHECK(ts->obj, ENODEV);
8937                 }
8938
8939                 /* ts->default_action_data. */
8940                 if (table->action_data_size_max) {
8941                         ts->default_action_data =
8942                                 malloc(table->action_data_size_max);
8943                         CHECK(ts->default_action_data, ENOMEM);
8944
8945                         memcpy(ts->default_action_data,
8946                                table->default_action_data,
8947                                table->action_data_size_max);
8948                 }
8949
8950                 /* ts->default_action_id. */
8951                 ts->default_action_id = table->default_action->id;
8952         }
8953
8954         TAILQ_FOREACH(s, &p->selectors, node) {
8955                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8956                 struct rte_swx_table_selector_params *params;
8957
8958                 /* ts->obj. */
8959                 params = selector_table_params_get(s);
8960                 CHECK(params, ENOMEM);
8961
8962                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8963
8964                 selector_params_free(params);
8965                 CHECK(ts->obj, ENODEV);
8966         }
8967
8968         TAILQ_FOREACH(l, &p->learners, node) {
8969                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8970                         p->n_selectors + l->id];
8971                 struct rte_swx_table_learner_params *params;
8972
8973                 /* ts->obj. */
8974                 params = learner_params_get(l);
8975                 CHECK(params, ENOMEM);
8976
8977                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8978                 learner_params_free(params);
8979                 CHECK(ts->obj, ENODEV);
8980
8981                 /* ts->default_action_data. */
8982                 if (l->action_data_size_max) {
8983                         ts->default_action_data = malloc(l->action_data_size_max);
8984                         CHECK(ts->default_action_data, ENOMEM);
8985
8986                         memcpy(ts->default_action_data,
8987                                l->default_action_data,
8988                                l->action_data_size_max);
8989                 }
8990
8991                 /* ts->default_action_id. */
8992                 ts->default_action_id = l->default_action->id;
8993         }
8994
8995         return 0;
8996 }
8997
8998 static void
8999 table_state_build_free(struct rte_swx_pipeline *p)
9000 {
9001         uint32_t i;
9002
9003         if (!p->table_state)
9004                 return;
9005
9006         for (i = 0; i < p->n_tables; i++) {
9007                 struct rte_swx_table_state *ts = &p->table_state[i];
9008                 struct table *table = table_find_by_id(p, i);
9009
9010                 /* ts->obj. */
9011                 if (table->type && ts->obj)
9012                         table->type->ops.free(ts->obj);
9013
9014                 /* ts->default_action_data. */
9015                 free(ts->default_action_data);
9016         }
9017
9018         for (i = 0; i < p->n_selectors; i++) {
9019                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9020
9021                 /* ts->obj. */
9022                 if (ts->obj)
9023                         rte_swx_table_selector_free(ts->obj);
9024         }
9025
9026         for (i = 0; i < p->n_learners; i++) {
9027                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9028
9029                 /* ts->obj. */
9030                 if (ts->obj)
9031                         rte_swx_table_learner_free(ts->obj);
9032
9033                 /* ts->default_action_data. */
9034                 free(ts->default_action_data);
9035         }
9036
9037         free(p->table_state);
9038         p->table_state = NULL;
9039 }
9040
9041 static void
9042 table_state_free(struct rte_swx_pipeline *p)
9043 {
9044         table_state_build_free(p);
9045 }
9046
9047 /*
9048  * Register array.
9049  */
9050 static struct regarray *
9051 regarray_find(struct rte_swx_pipeline *p, const char *name)
9052 {
9053         struct regarray *elem;
9054
9055         TAILQ_FOREACH(elem, &p->regarrays, node)
9056                 if (!strcmp(elem->name, name))
9057                         return elem;
9058
9059         return NULL;
9060 }
9061
9062 static struct regarray *
9063 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9064 {
9065         struct regarray *elem = NULL;
9066
9067         TAILQ_FOREACH(elem, &p->regarrays, node)
9068                 if (elem->id == id)
9069                         return elem;
9070
9071         return NULL;
9072 }
9073
9074 int
9075 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9076                               const char *name,
9077                               uint32_t size,
9078                               uint64_t init_val)
9079 {
9080         struct regarray *r;
9081
9082         CHECK(p, EINVAL);
9083
9084         CHECK_NAME(name, EINVAL);
9085         CHECK(!regarray_find(p, name), EEXIST);
9086
9087         CHECK(size, EINVAL);
9088         size = rte_align32pow2(size);
9089
9090         /* Memory allocation. */
9091         r = calloc(1, sizeof(struct regarray));
9092         CHECK(r, ENOMEM);
9093
9094         /* Node initialization. */
9095         strcpy(r->name, name);
9096         r->init_val = init_val;
9097         r->size = size;
9098         r->id = p->n_regarrays;
9099
9100         /* Node add to tailq. */
9101         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9102         p->n_regarrays++;
9103
9104         return 0;
9105 }
9106
9107 static int
9108 regarray_build(struct rte_swx_pipeline *p)
9109 {
9110         struct regarray *regarray;
9111
9112         if (!p->n_regarrays)
9113                 return 0;
9114
9115         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9116         CHECK(p->regarray_runtime, ENOMEM);
9117
9118         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9119                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9120                 uint32_t i;
9121
9122                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9123                                          RTE_CACHE_LINE_SIZE,
9124                                          p->numa_node);
9125                 CHECK(r->regarray, ENOMEM);
9126
9127                 if (regarray->init_val)
9128                         for (i = 0; i < regarray->size; i++)
9129                                 r->regarray[i] = regarray->init_val;
9130
9131                 r->size_mask = regarray->size - 1;
9132         }
9133
9134         return 0;
9135 }
9136
9137 static void
9138 regarray_build_free(struct rte_swx_pipeline *p)
9139 {
9140         uint32_t i;
9141
9142         if (!p->regarray_runtime)
9143                 return;
9144
9145         for (i = 0; i < p->n_regarrays; i++) {
9146                 struct regarray *regarray = regarray_find_by_id(p, i);
9147                 struct regarray_runtime *r = &p->regarray_runtime[i];
9148
9149                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9150         }
9151
9152         free(p->regarray_runtime);
9153         p->regarray_runtime = NULL;
9154 }
9155
9156 static void
9157 regarray_free(struct rte_swx_pipeline *p)
9158 {
9159         regarray_build_free(p);
9160
9161         for ( ; ; ) {
9162                 struct regarray *elem;
9163
9164                 elem = TAILQ_FIRST(&p->regarrays);
9165                 if (!elem)
9166                         break;
9167
9168                 TAILQ_REMOVE(&p->regarrays, elem, node);
9169                 free(elem);
9170         }
9171 }
9172
9173 /*
9174  * Meter array.
9175  */
9176 static struct meter_profile *
9177 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9178 {
9179         struct meter_profile *elem;
9180
9181         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9182                 if (!strcmp(elem->name, name))
9183                         return elem;
9184
9185         return NULL;
9186 }
9187
9188 static struct metarray *
9189 metarray_find(struct rte_swx_pipeline *p, const char *name)
9190 {
9191         struct metarray *elem;
9192
9193         TAILQ_FOREACH(elem, &p->metarrays, node)
9194                 if (!strcmp(elem->name, name))
9195                         return elem;
9196
9197         return NULL;
9198 }
9199
9200 static struct metarray *
9201 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9202 {
9203         struct metarray *elem = NULL;
9204
9205         TAILQ_FOREACH(elem, &p->metarrays, node)
9206                 if (elem->id == id)
9207                         return elem;
9208
9209         return NULL;
9210 }
9211
9212 int
9213 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9214                                  const char *name,
9215                                  uint32_t size)
9216 {
9217         struct metarray *m;
9218
9219         CHECK(p, EINVAL);
9220
9221         CHECK_NAME(name, EINVAL);
9222         CHECK(!metarray_find(p, name), EEXIST);
9223
9224         CHECK(size, EINVAL);
9225         size = rte_align32pow2(size);
9226
9227         /* Memory allocation. */
9228         m = calloc(1, sizeof(struct metarray));
9229         CHECK(m, ENOMEM);
9230
9231         /* Node initialization. */
9232         strcpy(m->name, name);
9233         m->size = size;
9234         m->id = p->n_metarrays;
9235
9236         /* Node add to tailq. */
9237         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9238         p->n_metarrays++;
9239
9240         return 0;
9241 }
9242
9243 struct meter_profile meter_profile_default = {
9244         .node = {0},
9245         .name = "",
9246         .params = {0},
9247
9248         .profile = {
9249                 .cbs = 10000,
9250                 .pbs = 10000,
9251                 .cir_period = 1,
9252                 .cir_bytes_per_period = 1,
9253                 .pir_period = 1,
9254                 .pir_bytes_per_period = 1,
9255         },
9256
9257         .n_users = 0,
9258 };
9259
9260 static void
9261 meter_init(struct meter *m)
9262 {
9263         memset(m, 0, sizeof(struct meter));
9264         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9265         m->profile = &meter_profile_default;
9266         m->color_mask = RTE_COLOR_GREEN;
9267
9268         meter_profile_default.n_users++;
9269 }
9270
9271 static int
9272 metarray_build(struct rte_swx_pipeline *p)
9273 {
9274         struct metarray *m;
9275
9276         if (!p->n_metarrays)
9277                 return 0;
9278
9279         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9280         CHECK(p->metarray_runtime, ENOMEM);
9281
9282         TAILQ_FOREACH(m, &p->metarrays, node) {
9283                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9284                 uint32_t i;
9285
9286                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9287                                          RTE_CACHE_LINE_SIZE,
9288                                          p->numa_node);
9289                 CHECK(r->metarray, ENOMEM);
9290
9291                 for (i = 0; i < m->size; i++)
9292                         meter_init(&r->metarray[i]);
9293
9294                 r->size_mask = m->size - 1;
9295         }
9296
9297         return 0;
9298 }
9299
9300 static void
9301 metarray_build_free(struct rte_swx_pipeline *p)
9302 {
9303         uint32_t i;
9304
9305         if (!p->metarray_runtime)
9306                 return;
9307
9308         for (i = 0; i < p->n_metarrays; i++) {
9309                 struct metarray *m = metarray_find_by_id(p, i);
9310                 struct metarray_runtime *r = &p->metarray_runtime[i];
9311
9312                 env_free(r->metarray, m->size * sizeof(struct meter));
9313         }
9314
9315         free(p->metarray_runtime);
9316         p->metarray_runtime = NULL;
9317 }
9318
9319 static void
9320 metarray_free(struct rte_swx_pipeline *p)
9321 {
9322         metarray_build_free(p);
9323
9324         /* Meter arrays. */
9325         for ( ; ; ) {
9326                 struct metarray *elem;
9327
9328                 elem = TAILQ_FIRST(&p->metarrays);
9329                 if (!elem)
9330                         break;
9331
9332                 TAILQ_REMOVE(&p->metarrays, elem, node);
9333                 free(elem);
9334         }
9335
9336         /* Meter profiles. */
9337         for ( ; ; ) {
9338                 struct meter_profile *elem;
9339
9340                 elem = TAILQ_FIRST(&p->meter_profiles);
9341                 if (!elem)
9342                         break;
9343
9344                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9345                 free(elem);
9346         }
9347 }
9348
9349 /*
9350  * Pipeline.
9351  */
9352 void
9353 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9354 {
9355         void *lib;
9356
9357         if (!p)
9358                 return;
9359
9360         lib = p->lib;
9361
9362         free(p->instruction_data);
9363         free(p->instructions);
9364
9365         metarray_free(p);
9366         regarray_free(p);
9367         table_state_free(p);
9368         learner_free(p);
9369         selector_free(p);
9370         table_free(p);
9371         action_free(p);
9372         instruction_table_free(p);
9373         metadata_free(p);
9374         header_free(p);
9375         extern_func_free(p);
9376         extern_obj_free(p);
9377         mirroring_free(p);
9378         port_out_free(p);
9379         port_in_free(p);
9380         struct_free(p);
9381
9382         free(p);
9383
9384         if (lib)
9385                 dlclose(lib);
9386 }
9387
9388 static int
9389 port_in_types_register(struct rte_swx_pipeline *p)
9390 {
9391         int status;
9392
9393         status = rte_swx_pipeline_port_in_type_register(p,
9394                 "ethdev",
9395                 &rte_swx_port_ethdev_reader_ops);
9396         if (status)
9397                 return status;
9398
9399         status = rte_swx_pipeline_port_in_type_register(p,
9400                 "ring",
9401                 &rte_swx_port_ring_reader_ops);
9402         if (status)
9403                 return status;
9404
9405 #ifdef RTE_PORT_PCAP
9406         status = rte_swx_pipeline_port_in_type_register(p,
9407                 "source",
9408                 &rte_swx_port_source_ops);
9409         if (status)
9410                 return status;
9411 #endif
9412
9413         status = rte_swx_pipeline_port_in_type_register(p,
9414                 "fd",
9415                 &rte_swx_port_fd_reader_ops);
9416         if (status)
9417                 return status;
9418
9419         return 0;
9420 }
9421
9422 static int
9423 port_out_types_register(struct rte_swx_pipeline *p)
9424 {
9425         int status;
9426
9427         status = rte_swx_pipeline_port_out_type_register(p,
9428                 "ethdev",
9429                 &rte_swx_port_ethdev_writer_ops);
9430         if (status)
9431                 return status;
9432
9433         status = rte_swx_pipeline_port_out_type_register(p,
9434                 "ring",
9435                 &rte_swx_port_ring_writer_ops);
9436         if (status)
9437                 return status;
9438
9439         status = rte_swx_pipeline_port_out_type_register(p,
9440                 "sink",
9441                 &rte_swx_port_sink_ops);
9442         if (status)
9443                 return status;
9444
9445         status = rte_swx_pipeline_port_out_type_register(p,
9446                 "fd",
9447                 &rte_swx_port_fd_writer_ops);
9448         if (status)
9449                 return status;
9450
9451         return 0;
9452 }
9453
9454 static int
9455 table_types_register(struct rte_swx_pipeline *p)
9456 {
9457         int status;
9458
9459         status = rte_swx_pipeline_table_type_register(p,
9460                 "exact",
9461                 RTE_SWX_TABLE_MATCH_EXACT,
9462                 &rte_swx_table_exact_match_ops);
9463         if (status)
9464                 return status;
9465
9466         status = rte_swx_pipeline_table_type_register(p,
9467                 "wildcard",
9468                 RTE_SWX_TABLE_MATCH_WILDCARD,
9469                 &rte_swx_table_wildcard_match_ops);
9470         if (status)
9471                 return status;
9472
9473         return 0;
9474 }
9475
9476 int
9477 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9478 {
9479         struct rte_swx_pipeline *pipeline = NULL;
9480         int status = 0;
9481
9482         /* Check input parameters. */
9483         CHECK(p, EINVAL);
9484
9485         /* Memory allocation. */
9486         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9487         if (!pipeline) {
9488                 status = -ENOMEM;
9489                 goto error;
9490         }
9491
9492         /* Initialization. */
9493         TAILQ_INIT(&pipeline->struct_types);
9494         TAILQ_INIT(&pipeline->port_in_types);
9495         TAILQ_INIT(&pipeline->ports_in);
9496         TAILQ_INIT(&pipeline->port_out_types);
9497         TAILQ_INIT(&pipeline->ports_out);
9498         TAILQ_INIT(&pipeline->extern_types);
9499         TAILQ_INIT(&pipeline->extern_objs);
9500         TAILQ_INIT(&pipeline->extern_funcs);
9501         TAILQ_INIT(&pipeline->headers);
9502         TAILQ_INIT(&pipeline->actions);
9503         TAILQ_INIT(&pipeline->table_types);
9504         TAILQ_INIT(&pipeline->tables);
9505         TAILQ_INIT(&pipeline->selectors);
9506         TAILQ_INIT(&pipeline->learners);
9507         TAILQ_INIT(&pipeline->regarrays);
9508         TAILQ_INIT(&pipeline->meter_profiles);
9509         TAILQ_INIT(&pipeline->metarrays);
9510
9511         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9512         pipeline->numa_node = numa_node;
9513
9514         status = port_in_types_register(pipeline);
9515         if (status)
9516                 goto error;
9517
9518         status = port_out_types_register(pipeline);
9519         if (status)
9520                 goto error;
9521
9522         status = table_types_register(pipeline);
9523         if (status)
9524                 goto error;
9525
9526         *p = pipeline;
9527         return 0;
9528
9529 error:
9530         rte_swx_pipeline_free(pipeline);
9531         return status;
9532 }
9533
9534 int
9535 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9536                                      const char **instructions,
9537                                      uint32_t n_instructions)
9538 {
9539         int err;
9540         uint32_t i;
9541
9542         err = instruction_config(p, NULL, instructions, n_instructions);
9543         if (err)
9544                 return err;
9545
9546         /* Thread instruction pointer reset. */
9547         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9548                 struct thread *t = &p->threads[i];
9549
9550                 thread_ip_reset(p, t);
9551         }
9552
9553         return 0;
9554 }
9555
9556 static int
9557 pipeline_compile(struct rte_swx_pipeline *p);
9558
9559 int
9560 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9561 {
9562         struct rte_swx_port_sink_params drop_port_params = {
9563                 .file_name = NULL,
9564         };
9565         int status;
9566
9567         CHECK(p, EINVAL);
9568         CHECK(p->build_done == 0, EEXIST);
9569
9570         status = port_in_build(p);
9571         if (status)
9572                 goto error;
9573
9574         /* Drop port. */
9575         status = rte_swx_pipeline_port_out_config(p,
9576                                                   p->n_ports_out,
9577                                                   "sink",
9578                                                   &drop_port_params);
9579         if (status)
9580                 goto error;
9581
9582         status = port_out_build(p);
9583         if (status)
9584                 goto error;
9585
9586         status = mirroring_build(p);
9587         if (status)
9588                 goto error;
9589
9590         status = struct_build(p);
9591         if (status)
9592                 goto error;
9593
9594         status = extern_obj_build(p);
9595         if (status)
9596                 goto error;
9597
9598         status = extern_func_build(p);
9599         if (status)
9600                 goto error;
9601
9602         status = header_build(p);
9603         if (status)
9604                 goto error;
9605
9606         status = metadata_build(p);
9607         if (status)
9608                 goto error;
9609
9610         status = instruction_table_build(p);
9611         if (status)
9612                 goto error;
9613
9614         status = action_build(p);
9615         if (status)
9616                 goto error;
9617
9618         status = table_build(p);
9619         if (status)
9620                 goto error;
9621
9622         status = selector_build(p);
9623         if (status)
9624                 goto error;
9625
9626         status = learner_build(p);
9627         if (status)
9628                 goto error;
9629
9630         status = table_state_build(p);
9631         if (status)
9632                 goto error;
9633
9634         status = regarray_build(p);
9635         if (status)
9636                 goto error;
9637
9638         status = metarray_build(p);
9639         if (status)
9640                 goto error;
9641
9642         p->build_done = 1;
9643
9644         pipeline_compile(p);
9645
9646         return 0;
9647
9648 error:
9649         metarray_build_free(p);
9650         regarray_build_free(p);
9651         table_state_build_free(p);
9652         learner_build_free(p);
9653         selector_build_free(p);
9654         table_build_free(p);
9655         action_build_free(p);
9656         instruction_table_build_free(p);
9657         metadata_build_free(p);
9658         header_build_free(p);
9659         extern_func_build_free(p);
9660         extern_obj_build_free(p);
9661         mirroring_build_free(p);
9662         port_out_build_free(p);
9663         port_in_build_free(p);
9664         struct_build_free(p);
9665
9666         return status;
9667 }
9668
9669 void
9670 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9671 {
9672         uint32_t i;
9673
9674         for (i = 0; i < n_instructions; i++)
9675                 instr_exec(p);
9676 }
9677
9678 void
9679 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9680 {
9681         uint32_t i;
9682
9683         for (i = 0; i < p->n_ports_out; i++) {
9684                 struct port_out_runtime *port = &p->out[i];
9685
9686                 if (port->flush)
9687                         port->flush(port->obj);
9688         }
9689 }
9690
9691 /*
9692  * Control.
9693  */
9694 int
9695 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9696                               struct rte_swx_ctl_pipeline_info *pipeline)
9697 {
9698         struct action *action;
9699         struct table *table;
9700         uint32_t n_actions = 0, n_tables = 0;
9701
9702         if (!p || !pipeline)
9703                 return -EINVAL;
9704
9705         TAILQ_FOREACH(action, &p->actions, node)
9706                 n_actions++;
9707
9708         TAILQ_FOREACH(table, &p->tables, node)
9709                 n_tables++;
9710
9711         pipeline->n_ports_in = p->n_ports_in;
9712         pipeline->n_ports_out = p->n_ports_out;
9713         pipeline->n_mirroring_slots = p->n_mirroring_slots;
9714         pipeline->n_mirroring_sessions = p->n_mirroring_sessions;
9715         pipeline->n_actions = n_actions;
9716         pipeline->n_tables = n_tables;
9717         pipeline->n_selectors = p->n_selectors;
9718         pipeline->n_learners = p->n_learners;
9719         pipeline->n_regarrays = p->n_regarrays;
9720         pipeline->n_metarrays = p->n_metarrays;
9721
9722         return 0;
9723 }
9724
9725 int
9726 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9727 {
9728         if (!p || !numa_node)
9729                 return -EINVAL;
9730
9731         *numa_node = p->numa_node;
9732         return 0;
9733 }
9734
9735 int
9736 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9737                             uint32_t action_id,
9738                             struct rte_swx_ctl_action_info *action)
9739 {
9740         struct action *a = NULL;
9741
9742         if (!p || (action_id >= p->n_actions) || !action)
9743                 return -EINVAL;
9744
9745         a = action_find_by_id(p, action_id);
9746         if (!a)
9747                 return -EINVAL;
9748
9749         strcpy(action->name, a->name);
9750         action->n_args = a->st ? a->st->n_fields : 0;
9751         return 0;
9752 }
9753
9754 int
9755 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9756                                 uint32_t action_id,
9757                                 uint32_t action_arg_id,
9758                                 struct rte_swx_ctl_action_arg_info *action_arg)
9759 {
9760         struct action *a = NULL;
9761         struct field *arg = NULL;
9762
9763         if (!p || (action_id >= p->n_actions) || !action_arg)
9764                 return -EINVAL;
9765
9766         a = action_find_by_id(p, action_id);
9767         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9768                 return -EINVAL;
9769
9770         arg = &a->st->fields[action_arg_id];
9771         strcpy(action_arg->name, arg->name);
9772         action_arg->n_bits = arg->n_bits;
9773         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9774
9775         return 0;
9776 }
9777
9778 int
9779 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9780                            uint32_t table_id,
9781                            struct rte_swx_ctl_table_info *table)
9782 {
9783         struct table *t = NULL;
9784
9785         if (!p || !table)
9786                 return -EINVAL;
9787
9788         t = table_find_by_id(p, table_id);
9789         if (!t)
9790                 return -EINVAL;
9791
9792         strcpy(table->name, t->name);
9793         strcpy(table->args, t->args);
9794         table->n_match_fields = t->n_fields;
9795         table->n_actions = t->n_actions;
9796         table->default_action_is_const = t->default_action_is_const;
9797         table->size = t->size;
9798         return 0;
9799 }
9800
9801 int
9802 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9803         uint32_t table_id,
9804         uint32_t match_field_id,
9805         struct rte_swx_ctl_table_match_field_info *match_field)
9806 {
9807         struct table *t;
9808         struct match_field *f;
9809
9810         if (!p || (table_id >= p->n_tables) || !match_field)
9811                 return -EINVAL;
9812
9813         t = table_find_by_id(p, table_id);
9814         if (!t || (match_field_id >= t->n_fields))
9815                 return -EINVAL;
9816
9817         f = &t->fields[match_field_id];
9818         match_field->match_type = f->match_type;
9819         match_field->is_header = t->header ? 1 : 0;
9820         match_field->n_bits = f->field->n_bits;
9821         match_field->offset = f->field->offset;
9822
9823         return 0;
9824 }
9825
9826 int
9827 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9828         uint32_t table_id,
9829         uint32_t table_action_id,
9830         struct rte_swx_ctl_table_action_info *table_action)
9831 {
9832         struct table *t;
9833
9834         if (!p || (table_id >= p->n_tables) || !table_action)
9835                 return -EINVAL;
9836
9837         t = table_find_by_id(p, table_id);
9838         if (!t || (table_action_id >= t->n_actions))
9839                 return -EINVAL;
9840
9841         table_action->action_id = t->actions[table_action_id]->id;
9842
9843         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9844         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9845
9846         return 0;
9847 }
9848
9849 int
9850 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9851                           uint32_t table_id,
9852                           struct rte_swx_table_ops *table_ops,
9853                           int *is_stub)
9854 {
9855         struct table *t;
9856
9857         if (!p || (table_id >= p->n_tables))
9858                 return -EINVAL;
9859
9860         t = table_find_by_id(p, table_id);
9861         if (!t)
9862                 return -EINVAL;
9863
9864         if (t->type) {
9865                 if (table_ops)
9866                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9867                 *is_stub = 0;
9868         } else {
9869                 *is_stub = 1;
9870         }
9871
9872         return 0;
9873 }
9874
9875 int
9876 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9877                               uint32_t selector_id,
9878                               struct rte_swx_ctl_selector_info *selector)
9879 {
9880         struct selector *s = NULL;
9881
9882         if (!p || !selector)
9883                 return -EINVAL;
9884
9885         s = selector_find_by_id(p, selector_id);
9886         if (!s)
9887                 return -EINVAL;
9888
9889         strcpy(selector->name, s->name);
9890
9891         selector->n_selector_fields = s->n_selector_fields;
9892         selector->n_groups_max = s->n_groups_max;
9893         selector->n_members_per_group_max = s->n_members_per_group_max;
9894
9895         return 0;
9896 }
9897
9898 int
9899 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9900          uint32_t selector_id,
9901          struct rte_swx_ctl_table_match_field_info *field)
9902 {
9903         struct selector *s;
9904
9905         if (!p || (selector_id >= p->n_selectors) || !field)
9906                 return -EINVAL;
9907
9908         s = selector_find_by_id(p, selector_id);
9909         if (!s)
9910                 return -EINVAL;
9911
9912         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9913         field->is_header = 0;
9914         field->n_bits = s->group_id_field->n_bits;
9915         field->offset = s->group_id_field->offset;
9916
9917         return 0;
9918 }
9919
9920 int
9921 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9922          uint32_t selector_id,
9923          uint32_t selector_field_id,
9924          struct rte_swx_ctl_table_match_field_info *field)
9925 {
9926         struct selector *s;
9927         struct field *f;
9928
9929         if (!p || (selector_id >= p->n_selectors) || !field)
9930                 return -EINVAL;
9931
9932         s = selector_find_by_id(p, selector_id);
9933         if (!s || (selector_field_id >= s->n_selector_fields))
9934                 return -EINVAL;
9935
9936         f = s->selector_fields[selector_field_id];
9937         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9938         field->is_header = s->selector_header ? 1 : 0;
9939         field->n_bits = f->n_bits;
9940         field->offset = f->offset;
9941
9942         return 0;
9943 }
9944
9945 int
9946 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9947          uint32_t selector_id,
9948          struct rte_swx_ctl_table_match_field_info *field)
9949 {
9950         struct selector *s;
9951
9952         if (!p || (selector_id >= p->n_selectors) || !field)
9953                 return -EINVAL;
9954
9955         s = selector_find_by_id(p, selector_id);
9956         if (!s)
9957                 return -EINVAL;
9958
9959         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9960         field->is_header = 0;
9961         field->n_bits = s->member_id_field->n_bits;
9962         field->offset = s->member_id_field->offset;
9963
9964         return 0;
9965 }
9966
9967 int
9968 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9969                              uint32_t learner_id,
9970                              struct rte_swx_ctl_learner_info *learner)
9971 {
9972         struct learner *l = NULL;
9973
9974         if (!p || !learner)
9975                 return -EINVAL;
9976
9977         l = learner_find_by_id(p, learner_id);
9978         if (!l)
9979                 return -EINVAL;
9980
9981         strcpy(learner->name, l->name);
9982
9983         learner->n_match_fields = l->n_fields;
9984         learner->n_actions = l->n_actions;
9985         learner->default_action_is_const = l->default_action_is_const;
9986         learner->size = l->size;
9987
9988         return 0;
9989 }
9990
9991 int
9992 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9993                                          uint32_t learner_id,
9994                                          uint32_t match_field_id,
9995                                          struct rte_swx_ctl_table_match_field_info *match_field)
9996 {
9997         struct learner *l;
9998         struct field *f;
9999
10000         if (!p || (learner_id >= p->n_learners) || !match_field)
10001                 return -EINVAL;
10002
10003         l = learner_find_by_id(p, learner_id);
10004         if (!l || (match_field_id >= l->n_fields))
10005                 return -EINVAL;
10006
10007         f = l->fields[match_field_id];
10008         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10009         match_field->is_header = l->header ? 1 : 0;
10010         match_field->n_bits = f->n_bits;
10011         match_field->offset = f->offset;
10012
10013         return 0;
10014 }
10015
10016 int
10017 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10018                                     uint32_t learner_id,
10019                                     uint32_t learner_action_id,
10020                                     struct rte_swx_ctl_table_action_info *learner_action)
10021 {
10022         struct learner *l;
10023
10024         if (!p || (learner_id >= p->n_learners) || !learner_action)
10025                 return -EINVAL;
10026
10027         l = learner_find_by_id(p, learner_id);
10028         if (!l || (learner_action_id >= l->n_actions))
10029                 return -EINVAL;
10030
10031         learner_action->action_id = l->actions[learner_action_id]->id;
10032
10033         learner_action->action_is_for_table_entries =
10034                 l->action_is_for_table_entries[learner_action_id];
10035
10036         learner_action->action_is_for_default_entry =
10037                 l->action_is_for_default_entry[learner_action_id];
10038
10039         return 0;
10040 }
10041
10042 int
10043 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10044                                  struct rte_swx_table_state **table_state)
10045 {
10046         if (!p || !table_state || !p->build_done)
10047                 return -EINVAL;
10048
10049         *table_state = p->table_state;
10050         return 0;
10051 }
10052
10053 int
10054 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10055                                  struct rte_swx_table_state *table_state)
10056 {
10057         if (!p || !table_state || !p->build_done)
10058                 return -EINVAL;
10059
10060         p->table_state = table_state;
10061         return 0;
10062 }
10063
10064 int
10065 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10066                                         uint32_t port_id,
10067                                         struct rte_swx_port_in_stats *stats)
10068 {
10069         struct port_in *port;
10070
10071         if (!p || !stats)
10072                 return -EINVAL;
10073
10074         port = port_in_find(p, port_id);
10075         if (!port)
10076                 return -EINVAL;
10077
10078         port->type->ops.stats_read(port->obj, stats);
10079         return 0;
10080 }
10081
10082 int
10083 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10084                                          uint32_t port_id,
10085                                          struct rte_swx_port_out_stats *stats)
10086 {
10087         struct port_out *port;
10088
10089         if (!p || !stats)
10090                 return -EINVAL;
10091
10092         port = port_out_find(p, port_id);
10093         if (!port)
10094                 return -EINVAL;
10095
10096         port->type->ops.stats_read(port->obj, stats);
10097         return 0;
10098 }
10099
10100 int
10101 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10102                                       const char *table_name,
10103                                       struct rte_swx_table_stats *stats)
10104 {
10105         struct table *table;
10106         struct table_statistics *table_stats;
10107
10108         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10109                 return -EINVAL;
10110
10111         table = table_find(p, table_name);
10112         if (!table)
10113                 return -EINVAL;
10114
10115         table_stats = &p->table_stats[table->id];
10116
10117         memcpy(stats->n_pkts_action,
10118                table_stats->n_pkts_action,
10119                p->n_actions * sizeof(uint64_t));
10120
10121         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10122         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10123
10124         return 0;
10125 }
10126
10127 int
10128 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10129         const char *selector_name,
10130         struct rte_swx_pipeline_selector_stats *stats)
10131 {
10132         struct selector *s;
10133
10134         if (!p || !selector_name || !selector_name[0] || !stats)
10135                 return -EINVAL;
10136
10137         s = selector_find(p, selector_name);
10138         if (!s)
10139                 return -EINVAL;
10140
10141         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10142
10143         return 0;
10144 }
10145
10146 int
10147 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10148                                         const char *learner_name,
10149                                         struct rte_swx_learner_stats *stats)
10150 {
10151         struct learner *l;
10152         struct learner_statistics *learner_stats;
10153
10154         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10155                 return -EINVAL;
10156
10157         l = learner_find(p, learner_name);
10158         if (!l)
10159                 return -EINVAL;
10160
10161         learner_stats = &p->learner_stats[l->id];
10162
10163         memcpy(stats->n_pkts_action,
10164                learner_stats->n_pkts_action,
10165                p->n_actions * sizeof(uint64_t));
10166
10167         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10168         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10169
10170         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10171         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10172
10173         stats->n_pkts_forget = learner_stats->n_pkts_forget;
10174
10175         return 0;
10176 }
10177
10178 int
10179 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10180                               uint32_t regarray_id,
10181                               struct rte_swx_ctl_regarray_info *regarray)
10182 {
10183         struct regarray *r;
10184
10185         if (!p || !regarray)
10186                 return -EINVAL;
10187
10188         r = regarray_find_by_id(p, regarray_id);
10189         if (!r)
10190                 return -EINVAL;
10191
10192         strcpy(regarray->name, r->name);
10193         regarray->size = r->size;
10194         return 0;
10195 }
10196
10197 int
10198 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10199                                    const char *regarray_name,
10200                                    uint32_t regarray_index,
10201                                    uint64_t *value)
10202 {
10203         struct regarray *regarray;
10204         struct regarray_runtime *r;
10205
10206         if (!p || !regarray_name || !value)
10207                 return -EINVAL;
10208
10209         regarray = regarray_find(p, regarray_name);
10210         if (!regarray || (regarray_index >= regarray->size))
10211                 return -EINVAL;
10212
10213         r = &p->regarray_runtime[regarray->id];
10214         *value = r->regarray[regarray_index];
10215         return 0;
10216 }
10217
10218 int
10219 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10220                                    const char *regarray_name,
10221                                    uint32_t regarray_index,
10222                                    uint64_t value)
10223 {
10224         struct regarray *regarray;
10225         struct regarray_runtime *r;
10226
10227         if (!p || !regarray_name)
10228                 return -EINVAL;
10229
10230         regarray = regarray_find(p, regarray_name);
10231         if (!regarray || (regarray_index >= regarray->size))
10232                 return -EINVAL;
10233
10234         r = &p->regarray_runtime[regarray->id];
10235         r->regarray[regarray_index] = value;
10236         return 0;
10237 }
10238
10239 int
10240 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10241                               uint32_t metarray_id,
10242                               struct rte_swx_ctl_metarray_info *metarray)
10243 {
10244         struct metarray *m;
10245
10246         if (!p || !metarray)
10247                 return -EINVAL;
10248
10249         m = metarray_find_by_id(p, metarray_id);
10250         if (!m)
10251                 return -EINVAL;
10252
10253         strcpy(metarray->name, m->name);
10254         metarray->size = m->size;
10255         return 0;
10256 }
10257
10258 int
10259 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10260                               const char *name,
10261                               struct rte_meter_trtcm_params *params)
10262 {
10263         struct meter_profile *mp;
10264         int status;
10265
10266         CHECK(p, EINVAL);
10267         CHECK_NAME(name, EINVAL);
10268         CHECK(params, EINVAL);
10269         CHECK(!meter_profile_find(p, name), EEXIST);
10270
10271         /* Node allocation. */
10272         mp = calloc(1, sizeof(struct meter_profile));
10273         CHECK(mp, ENOMEM);
10274
10275         /* Node initialization. */
10276         strcpy(mp->name, name);
10277         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10278         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10279         if (status) {
10280                 free(mp);
10281                 CHECK(0, EINVAL);
10282         }
10283
10284         /* Node add to tailq. */
10285         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10286
10287         return 0;
10288 }
10289
10290 int
10291 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10292                                  const char *name)
10293 {
10294         struct meter_profile *mp;
10295
10296         CHECK(p, EINVAL);
10297         CHECK_NAME(name, EINVAL);
10298
10299         mp = meter_profile_find(p, name);
10300         CHECK(mp, EINVAL);
10301         CHECK(!mp->n_users, EBUSY);
10302
10303         /* Remove node from tailq. */
10304         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10305         free(mp);
10306
10307         return 0;
10308 }
10309
10310 int
10311 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10312                         const char *metarray_name,
10313                         uint32_t metarray_index)
10314 {
10315         struct meter_profile *mp_old;
10316         struct metarray *metarray;
10317         struct metarray_runtime *metarray_runtime;
10318         struct meter *m;
10319
10320         CHECK(p, EINVAL);
10321         CHECK_NAME(metarray_name, EINVAL);
10322
10323         metarray = metarray_find(p, metarray_name);
10324         CHECK(metarray, EINVAL);
10325         CHECK(metarray_index < metarray->size, EINVAL);
10326
10327         metarray_runtime = &p->metarray_runtime[metarray->id];
10328         m = &metarray_runtime->metarray[metarray_index];
10329         mp_old = m->profile;
10330
10331         meter_init(m);
10332
10333         mp_old->n_users--;
10334
10335         return 0;
10336 }
10337
10338 int
10339 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10340                       const char *metarray_name,
10341                       uint32_t metarray_index,
10342                       const char *profile_name)
10343 {
10344         struct meter_profile *mp, *mp_old;
10345         struct metarray *metarray;
10346         struct metarray_runtime *metarray_runtime;
10347         struct meter *m;
10348
10349         CHECK(p, EINVAL);
10350         CHECK_NAME(metarray_name, EINVAL);
10351
10352         metarray = metarray_find(p, metarray_name);
10353         CHECK(metarray, EINVAL);
10354         CHECK(metarray_index < metarray->size, EINVAL);
10355
10356         mp = meter_profile_find(p, profile_name);
10357         CHECK(mp, EINVAL);
10358
10359         metarray_runtime = &p->metarray_runtime[metarray->id];
10360         m = &metarray_runtime->metarray[metarray_index];
10361         mp_old = m->profile;
10362
10363         memset(m, 0, sizeof(struct meter));
10364         rte_meter_trtcm_config(&m->m, &mp->profile);
10365         m->profile = mp;
10366         m->color_mask = RTE_COLORS;
10367
10368         mp->n_users++;
10369         mp_old->n_users--;
10370
10371         return 0;
10372 }
10373
10374 int
10375 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10376                              const char *metarray_name,
10377                              uint32_t metarray_index,
10378                              struct rte_swx_ctl_meter_stats *stats)
10379 {
10380         struct metarray *metarray;
10381         struct metarray_runtime *metarray_runtime;
10382         struct meter *m;
10383
10384         CHECK(p, EINVAL);
10385         CHECK_NAME(metarray_name, EINVAL);
10386
10387         metarray = metarray_find(p, metarray_name);
10388         CHECK(metarray, EINVAL);
10389         CHECK(metarray_index < metarray->size, EINVAL);
10390
10391         CHECK(stats, EINVAL);
10392
10393         metarray_runtime = &p->metarray_runtime[metarray->id];
10394         m = &metarray_runtime->metarray[metarray_index];
10395
10396         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10397         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10398
10399         return 0;
10400 }
10401
10402 int
10403 rte_swx_ctl_pipeline_mirroring_session_set(struct rte_swx_pipeline *p,
10404                                            uint32_t session_id,
10405                                            struct rte_swx_pipeline_mirroring_session_params *params)
10406 {
10407         struct mirroring_session *s;
10408
10409         CHECK(p, EINVAL);
10410         CHECK(p->build_done, EEXIST);
10411         CHECK(session_id < p->n_mirroring_sessions, EINVAL);
10412         CHECK(params, EINVAL);
10413         CHECK(params->port_id < p->n_ports_out, EINVAL);
10414
10415         s = &p->mirroring_sessions[session_id];
10416         s->port_id = params->port_id;
10417         s->fast_clone = params->fast_clone;
10418         s->truncation_length = params->truncation_length ? params->truncation_length : UINT32_MAX;
10419
10420         return 0;
10421 }
10422
10423 /*
10424  * Pipeline compilation.
10425  */
10426 static const char *
10427 instr_type_to_name(struct instruction *instr)
10428 {
10429         switch (instr->type) {
10430         case INSTR_RX: return "INSTR_RX";
10431
10432         case INSTR_TX: return "INSTR_TX";
10433         case INSTR_TX_I: return "INSTR_TX_I";
10434         case INSTR_DROP: return "INSTR_DROP";
10435         case INSTR_MIRROR: return "INSTR_MIRROR";
10436         case INSTR_RECIRCULATE: return "INSTR_RECIRCULATE";
10437         case INSTR_RECIRCID: return "INSTR_RECIRCID";
10438
10439         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10440         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10441         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10442         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10443         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10444         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10445         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10446         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10447
10448         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10449
10450         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10451
10452         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10453         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10454         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10455         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10456         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10457         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10458         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10459         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10460         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10461
10462         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10463         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10464
10465         case INSTR_MOV: return "INSTR_MOV";
10466         case INSTR_MOV_MH: return "INSTR_MOV_MH";
10467         case INSTR_MOV_HM: return "INSTR_MOV_HM";
10468         case INSTR_MOV_HH: return "INSTR_MOV_HH";
10469         case INSTR_MOV_I: return "INSTR_MOV_I";
10470
10471         case INSTR_DMA_HT: return "INSTR_DMA_HT";
10472         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10473         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10474         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10475         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10476         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10477         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10478         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10479
10480         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10481         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10482         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10483         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10484         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10485         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10486
10487         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10488         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10489         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10490         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10491         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10492         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10493
10494         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10495         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10496         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10497         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10498
10499         case INSTR_ALU_AND: return "INSTR_ALU_AND";
10500         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10501         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10502         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10503         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10504
10505         case INSTR_ALU_OR: return "INSTR_ALU_OR";
10506         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10507         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10508         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10509         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10510
10511         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10512         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10513         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10514         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10515         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10516
10517         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10518         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10519         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10520         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10521         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10522         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10523
10524         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10525         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10526         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10527         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10528         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10529         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10530
10531         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10532         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10533         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10534
10535         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10536         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10537         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10538         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10539         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10540         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10541
10542         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10543         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10544         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10545         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10546         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10547         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10548         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10549         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10550         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10551
10552         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10553         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10554         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10555         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10556         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10557         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10558         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10559         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10560         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10561
10562         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10563         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10564         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10565
10566         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10567         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10568         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10569         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10570         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10571         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10572         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10573         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10574         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10575         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10576         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10577         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10578
10579         case INSTR_TABLE: return "INSTR_TABLE";
10580         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10581         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10582         case INSTR_LEARNER: return "INSTR_LEARNER";
10583         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10584
10585         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10586         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10587
10588         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10589         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10590
10591         case INSTR_JMP: return "INSTR_JMP";
10592         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10593         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10594         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10595         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10596         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10597         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10598         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10599         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10600         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10601         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10602         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10603         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10604         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10605         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10606         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10607         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10608         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10609         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10610         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10611         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10612         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10613         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10614         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10615         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10616         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10617         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10618         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10619         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10620
10621         case INSTR_RETURN: return "INSTR_RETURN";
10622
10623         default: return "INSTR_UNKNOWN";
10624         }
10625 }
10626
10627 typedef void
10628 (*instruction_export_t)(struct instruction *, FILE *);
10629
10630 static void
10631 instr_io_export(struct instruction *instr, FILE *f)
10632 {
10633         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10634
10635         /* n_io, n_io_imm, n_hdrs. */
10636         if (instr->type == INSTR_RX ||
10637             instr->type == INSTR_TX ||
10638             instr->type == INSTR_HDR_EXTRACT_M ||
10639             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10640                 n_io = 1;
10641
10642         if (instr->type == INSTR_TX_I)
10643                 n_io_imm = 1;
10644
10645         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10646                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10647
10648         if (instr->type == INSTR_HDR_EXTRACT_M ||
10649             instr->type == INSTR_HDR_LOOKAHEAD ||
10650             instr->type == INSTR_HDR_EMIT)
10651                 n_hdrs = 1;
10652
10653         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10654                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10655
10656         /* instr. */
10657         fprintf(f,
10658                 "\t{\n"
10659                 "\t\t.type = %s,\n",
10660                 instr_type_to_name(instr));
10661
10662         /* instr.io. */
10663         if (n_io || n_io_imm || n_hdrs)
10664                 fprintf(f,
10665                         "\t\t.io = {\n");
10666
10667         /* instr.io.io. */
10668         if (n_io)
10669                 fprintf(f,
10670                         "\t\t\t.io = {\n"
10671                         "\t\t\t\t.offset = %u,\n"
10672                         "\t\t\t\t.n_bits = %u,\n"
10673                         "\t\t\t},\n",
10674                         instr->io.io.offset,
10675                         instr->io.io.n_bits);
10676
10677         if (n_io_imm)
10678                 fprintf(f,
10679                         "\t\t\t.io = {\n"
10680                         "\t\t\t\t.val = %u,\n"
10681                         "\t\t\t},\n",
10682                         instr->io.io.val);
10683
10684         /* instr.io.hdr. */
10685         if (n_hdrs) {
10686                 fprintf(f,
10687                         "\t\t.hdr = {\n");
10688
10689                 /* instr.io.hdr.header_id. */
10690                 fprintf(f,
10691                         "\t\t\t.header_id = {");
10692
10693                 for (i = 0; i < n_hdrs; i++)
10694                         fprintf(f,
10695                                 "%u, ",
10696                                 instr->io.hdr.header_id[i]);
10697
10698                 fprintf(f,
10699                         "},\n");
10700
10701                 /* instr.io.hdr.struct_id. */
10702                 fprintf(f,
10703                         "\t\t\t.struct_id = {");
10704
10705                 for (i = 0; i < n_hdrs; i++)
10706                         fprintf(f,
10707                                 "%u, ",
10708                                 instr->io.hdr.struct_id[i]);
10709
10710                 fprintf(f,
10711                         "},\n");
10712
10713                 /* instr.io.hdr.n_bytes. */
10714                 fprintf(f,
10715                         "\t\t\t.n_bytes = {");
10716
10717                 for (i = 0; i < n_hdrs; i++)
10718                         fprintf(f,
10719                                 "%u, ",
10720                                 instr->io.hdr.n_bytes[i]);
10721
10722                 fprintf(f,
10723                         "},\n");
10724
10725                 /* instr.io.hdr - closing curly brace. */
10726                 fprintf(f,
10727                         "\t\t\t}\n,");
10728         }
10729
10730         /* instr.io - closing curly brace. */
10731         if (n_io || n_io_imm || n_hdrs)
10732                 fprintf(f,
10733                         "\t\t},\n");
10734
10735         /* instr - closing curly brace. */
10736         fprintf(f,
10737                 "\t},\n");
10738 }
10739
10740 static void
10741 instr_mirror_export(struct instruction *instr, FILE *f)
10742 {
10743         fprintf(f,
10744                 "\t{\n"
10745                 "\t\t.type = %s,\n"
10746                 "\t\t.mirror = {\n"
10747                 "\t\t\t.dst = {\n"
10748                 "\t\t\t\t.struct_id = %u,\n"
10749                 "\t\t\t\t.n_bits = %u,\n"
10750                 "\t\t\t\t.offset = %u,\n"
10751                 "\t\t\t}\n,"
10752                 "\t\t\t.src = {\n"
10753                 "\t\t\t\t.struct_id = %u,\n"
10754                 "\t\t\t\t.n_bits = %u,\n"
10755                 "\t\t\t\t.offset = %u,\n"
10756                 "\t\t\t}\n,"
10757                 "\t\t},\n"
10758                 "\t},\n",
10759                 instr_type_to_name(instr),
10760                 instr->mirror.dst.struct_id,
10761                 instr->mirror.dst.n_bits,
10762                 instr->mirror.dst.offset,
10763                 instr->mirror.src.struct_id,
10764                 instr->mirror.src.n_bits,
10765                 instr->mirror.src.offset);
10766 }
10767
10768 static void
10769 instr_recirculate_export(struct instruction *instr, FILE *f)
10770 {
10771         fprintf(f,
10772                 "\t{\n"
10773                 "\t\t.type = %s,\n"
10774                 "\t},\n",
10775                 instr_type_to_name(instr));
10776 }
10777
10778 static void
10779 instr_recircid_export(struct instruction *instr, FILE *f)
10780 {
10781         fprintf(f,
10782                 "\t{\n"
10783                 "\t\t.type = %s,\n"
10784                 "\t\t.io = {\n"
10785                 "\t\t\t.offset = %u,\n"
10786                 "\t\t\t.n_bits = %u,\n"
10787                 "\t\t},\n"
10788                 "\t},\n",
10789                 instr_type_to_name(instr),
10790                 instr->io.io.offset,
10791                 instr->io.io.n_bits);
10792 }
10793
10794 static void
10795 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10796 {
10797         fprintf(f,
10798                 "\t{\n"
10799                 "\t\t.type = %s,\n"
10800                 "\t\t.valid = {\n"
10801                 "\t\t\t.header_id = %u,\n"
10802                 "\t\t},\n"
10803                 "\t},\n",
10804                 instr_type_to_name(instr),
10805                 instr->valid.header_id);
10806 }
10807
10808 static void
10809 instr_mov_export(struct instruction *instr, FILE *f)
10810 {
10811         if (instr->type != INSTR_MOV_I)
10812                 fprintf(f,
10813                         "\t{\n"
10814                         "\t\t.type = %s,\n"
10815                         "\t\t.mov = {\n"
10816                         "\t\t\t.dst = {\n"
10817                         "\t\t\t\t.struct_id = %u,\n"
10818                         "\t\t\t\t.n_bits = %u,\n"
10819                         "\t\t\t\t.offset = %u,\n"
10820                         "\t\t\t},\n"
10821                         "\t\t\t.src = {\n"
10822                         "\t\t\t\t.struct_id = %u,\n"
10823                         "\t\t\t\t.n_bits = %u,\n"
10824                         "\t\t\t\t.offset = %u,\n"
10825                         "\t\t\t},\n"
10826                         "\t\t},\n"
10827                         "\t},\n",
10828                         instr_type_to_name(instr),
10829                         instr->mov.dst.struct_id,
10830                         instr->mov.dst.n_bits,
10831                         instr->mov.dst.offset,
10832                         instr->mov.src.struct_id,
10833                         instr->mov.src.n_bits,
10834                         instr->mov.src.offset);
10835         else
10836                 fprintf(f,
10837                         "\t{\n"
10838                         "\t\t.type = %s,\n"
10839                         "\t\t.mov = {\n"
10840                         "\t\t\t.dst = {\n"
10841                         "\t\t\t\t.struct_id = %u,\n"
10842                         "\t\t\t\t.n_bits = %u,\n"
10843                         "\t\t\t\t.offset = %u,\n"
10844                         "\t\t\t}\n,"
10845                         "\t\t\t.src_val = %" PRIu64 ",\n"
10846                         "\t\t},\n"
10847                         "\t},\n",
10848                         instr_type_to_name(instr),
10849                         instr->mov.dst.struct_id,
10850                         instr->mov.dst.n_bits,
10851                         instr->mov.dst.offset,
10852                         instr->mov.src_val);
10853 }
10854
10855 static void
10856 instr_dma_ht_export(struct instruction *instr, FILE *f)
10857 {
10858         uint32_t n_dma = 0, i;
10859
10860         /* n_dma. */
10861         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10862
10863         /* instr. */
10864         fprintf(f,
10865                 "\t{\n"
10866                 "\t\t.type = %s,\n",
10867                 instr_type_to_name(instr));
10868
10869         /* instr.dma. */
10870         fprintf(f,
10871                 "\t\t.dma = {\n");
10872
10873         /* instr.dma.dst. */
10874         fprintf(f,
10875                 "\t\t\t.dst = {\n");
10876
10877         /* instr.dma.dst.header_id. */
10878         fprintf(f,
10879                 "\t\t\t\t.header_id = {");
10880
10881         for (i = 0; i < n_dma; i++)
10882                 fprintf(f,
10883                         "%u, ",
10884                         instr->dma.dst.header_id[i]);
10885
10886         fprintf(f,
10887                 "},\n");
10888
10889         /* instr.dma.dst.struct_id. */
10890         fprintf(f,
10891                 "\t\t\t\t.struct_id = {");
10892
10893         for (i = 0; i < n_dma; i++)
10894                 fprintf(f,
10895                         "%u, ",
10896                         instr->dma.dst.struct_id[i]);
10897
10898         fprintf(f,
10899                 "},\n");
10900
10901         /* instr.dma.dst - closing curly brace. */
10902         fprintf(f,
10903                 "\t\t\t},\n");
10904
10905         /* instr.dma.src. */
10906         fprintf(f,
10907                 "\t\t\t.src = {\n");
10908
10909         /* instr.dma.src.offset. */
10910         fprintf(f,
10911                 "\t\t\t\t.offset = {");
10912
10913         for (i = 0; i < n_dma; i++)
10914                 fprintf(f,
10915                         "%u, ",
10916                         instr->dma.src.offset[i]);
10917
10918         fprintf(f,
10919                 "},\n");
10920
10921         /* instr.dma.src - closing curly brace. */
10922         fprintf(f,
10923                 "\t\t\t},\n");
10924
10925         /* instr.dma.n_bytes. */
10926         fprintf(f,
10927                 "\t\t\t.n_bytes = {");
10928
10929         for (i = 0; i < n_dma; i++)
10930                 fprintf(f,
10931                         "%u, ",
10932                         instr->dma.n_bytes[i]);
10933
10934         fprintf(f,
10935                 "},\n");
10936
10937         /* instr.dma - closing curly brace. */
10938         fprintf(f,
10939                 "\t\t},\n");
10940
10941         /* instr - closing curly brace. */
10942         fprintf(f,
10943                 "\t},\n");
10944 }
10945
10946 static void
10947 instr_alu_export(struct instruction *instr, FILE *f)
10948 {
10949         int imm = 0;
10950
10951         if (instr->type == INSTR_ALU_ADD_MI ||
10952             instr->type == INSTR_ALU_ADD_HI ||
10953             instr->type == INSTR_ALU_SUB_MI ||
10954             instr->type == INSTR_ALU_SUB_HI ||
10955             instr->type == INSTR_ALU_SHL_MI ||
10956             instr->type == INSTR_ALU_SHL_HI ||
10957             instr->type == INSTR_ALU_SHR_MI ||
10958             instr->type == INSTR_ALU_SHR_HI ||
10959             instr->type == INSTR_ALU_AND_I ||
10960             instr->type == INSTR_ALU_OR_I ||
10961             instr->type == INSTR_ALU_XOR_I)
10962                 imm = 1;
10963
10964         if (!imm)
10965                 fprintf(f,
10966                         "\t{\n"
10967                         "\t\t.type = %s,\n"
10968                         "\t\t.alu = {\n"
10969                         "\t\t\t.dst = {\n"
10970                         "\t\t\t\t.struct_id = %u,\n"
10971                         "\t\t\t\t.n_bits = %u,\n"
10972                         "\t\t\t\t.offset = %u,\n"
10973                         "\t\t\t},\n"
10974                         "\t\t\t.src = {\n"
10975                         "\t\t\t\t.struct_id = %u,\n"
10976                         "\t\t\t\t.n_bits = %u,\n"
10977                         "\t\t\t\t.offset = %u,\n"
10978                         "\t\t\t},\n"
10979                         "\t\t},\n"
10980                         "\t},\n",
10981                         instr_type_to_name(instr),
10982                         instr->alu.dst.struct_id,
10983                         instr->alu.dst.n_bits,
10984                         instr->alu.dst.offset,
10985                         instr->alu.src.struct_id,
10986                         instr->alu.src.n_bits,
10987                         instr->alu.src.offset);
10988         else
10989                 fprintf(f,
10990                         "\t{\n"
10991                         "\t\t.type = %s,\n"
10992                         "\t\t.alu = {\n"
10993                         "\t\t\t.dst = {\n"
10994                         "\t\t\t\t.struct_id = %u,\n"
10995                         "\t\t\t\t.n_bits = %u,\n"
10996                         "\t\t\t\t.offset = %u,\n"
10997                         "\t\t\t}\n,"
10998                         "\t\t\t.src_val = %" PRIu64 ",\n"
10999                         "\t\t},\n"
11000                         "\t},\n",
11001                         instr_type_to_name(instr),
11002                         instr->alu.dst.struct_id,
11003                         instr->alu.dst.n_bits,
11004                         instr->alu.dst.offset,
11005                         instr->alu.src_val);
11006 }
11007
11008 static void
11009 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11010 {
11011         int prefetch  = 0, idx_imm = 0, src_imm = 0;
11012
11013         if (instr->type == INSTR_REGPREFETCH_RH ||
11014             instr->type == INSTR_REGPREFETCH_RM ||
11015             instr->type == INSTR_REGPREFETCH_RI)
11016                 prefetch = 1;
11017
11018         /* index is the 3rd operand for the regrd instruction and the 2nd
11019          * operand for the regwr and regadd instructions.
11020          */
11021         if (instr->type == INSTR_REGPREFETCH_RI ||
11022             instr->type == INSTR_REGRD_HRI ||
11023             instr->type == INSTR_REGRD_MRI ||
11024             instr->type == INSTR_REGWR_RIH ||
11025             instr->type == INSTR_REGWR_RIM ||
11026             instr->type == INSTR_REGWR_RII ||
11027             instr->type == INSTR_REGADD_RIH ||
11028             instr->type == INSTR_REGADD_RIM ||
11029             instr->type == INSTR_REGADD_RII)
11030                 idx_imm = 1;
11031
11032         /* src is the 3rd operand for the regwr and regadd instructions. */
11033         if (instr->type == INSTR_REGWR_RHI ||
11034             instr->type == INSTR_REGWR_RMI ||
11035             instr->type == INSTR_REGWR_RII ||
11036             instr->type == INSTR_REGADD_RHI ||
11037             instr->type == INSTR_REGADD_RMI ||
11038             instr->type == INSTR_REGADD_RII)
11039                 src_imm = 1;
11040
11041         /* instr.regarray.regarray_id. */
11042         fprintf(f,
11043                 "\t{\n"
11044                 "\t\t.type = %s,\n"
11045                 "\t\t.regarray = {\n"
11046                 "\t\t\t.regarray_id = %u,\n",
11047                 instr_type_to_name(instr),
11048                 instr->regarray.regarray_id);
11049
11050         /* instr.regarray.idx / instr.regarray.idx_val. */
11051         if (!idx_imm)
11052                 fprintf(f,
11053                         "\t\t\t\t.idx = {\n"
11054                         "\t\t\t\t\t.struct_id = %u,\n"
11055                         "\t\t\t\t\t.n_bits = %u,\n"
11056                         "\t\t\t\t\t.offset = %u,\n"
11057                         "\t\t\t\t},\n",
11058                         instr->regarray.idx.struct_id,
11059                         instr->regarray.idx.n_bits,
11060                         instr->regarray.idx.offset);
11061         else
11062                 fprintf(f,
11063                         "\t\t\t\t.idx_val = %u,\n",
11064                         instr->regarray.idx_val);
11065
11066         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
11067         if (!prefetch) {
11068                 if (!src_imm)
11069                         fprintf(f,
11070                                 "\t\t\t\t.dstsrc = {\n"
11071                                 "\t\t\t\t\t.struct_id = %u,\n"
11072                                 "\t\t\t\t\t.n_bits = %u,\n"
11073                                 "\t\t\t\t\t.offset = %u,\n"
11074                                 "\t\t\t\t},\n",
11075                                 instr->regarray.dstsrc.struct_id,
11076                                 instr->regarray.dstsrc.n_bits,
11077                                 instr->regarray.dstsrc.offset);
11078                 else
11079                         fprintf(f,
11080                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
11081                                 instr->regarray.dstsrc_val);
11082         }
11083
11084         /* instr.regarray and instr - closing curly braces. */
11085         fprintf(f,
11086                 "\t\t},\n"
11087                 "\t},\n");
11088 }
11089
11090 static void
11091 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11092 {
11093         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
11094
11095         if (instr->type == INSTR_METPREFETCH_H ||
11096             instr->type == INSTR_METPREFETCH_M ||
11097             instr->type == INSTR_METPREFETCH_I)
11098                 prefetch = 1;
11099
11100         /* idx_imm. */
11101         if (instr->type == INSTR_METPREFETCH_I ||
11102             instr->type == INSTR_METER_IHM ||
11103             instr->type == INSTR_METER_IHI ||
11104             instr->type == INSTR_METER_IMM ||
11105             instr->type == INSTR_METER_IMI)
11106                 idx_imm = 1;
11107
11108         /* color_in_imm. */
11109         if (instr->type == INSTR_METER_HHI ||
11110             instr->type == INSTR_METER_HMI ||
11111             instr->type == INSTR_METER_MHI ||
11112             instr->type == INSTR_METER_MMI ||
11113             instr->type == INSTR_METER_IHI ||
11114             instr->type == INSTR_METER_IMI)
11115                 color_in_imm = 1;
11116
11117         /* instr.meter.metarray_id. */
11118         fprintf(f,
11119                 "\t{\n"
11120                 "\t\t.type = %s,\n"
11121                 "\t\t.meter = {\n"
11122                 "\t\t\t.metarray_id = %u,\n",
11123                 instr_type_to_name(instr),
11124                 instr->meter.metarray_id);
11125
11126         /* instr.meter.idx / instr.meter.idx_val. */
11127         if (!idx_imm)
11128                 fprintf(f,
11129                         "\t\t\t.idx = {\n"
11130                         "\t\t\t\t.struct_id = %u,\n"
11131                         "\t\t\t\t.n_bits = %u,\n"
11132                         "\t\t\t\t.offset = %u,\n"
11133                         "\t\t\t},\n",
11134                         instr->meter.idx.struct_id,
11135                         instr->meter.idx.n_bits,
11136                         instr->meter.idx.offset);
11137         else
11138                 fprintf(f,
11139                         "\t\t\t.idx_val = %u,\n",
11140                         instr->meter.idx_val);
11141
11142         if (!prefetch) {
11143                 /* instr.meter.length. */
11144                 fprintf(f,
11145                         "\t\t\t.length = {\n"
11146                         "\t\t\t\t.struct_id = %u,\n"
11147                         "\t\t\t\t.n_bits = %u,\n"
11148                         "\t\t\t\t.offset = %u,\n"
11149                         "\t\t\t},\n",
11150                         instr->meter.length.struct_id,
11151                         instr->meter.length.n_bits,
11152                         instr->meter.length.offset);
11153
11154                 /* instr.meter.color_in / instr.meter.color_in_val. */
11155                 if (!color_in_imm)
11156                         fprintf(f,
11157                                 "\t\t\t.color_in = {\n"
11158                                 "\t\t\t\t.struct_id = %u,\n"
11159                                 "\t\t\t\t.n_bits = %u,\n"
11160                                 "\t\t\t\t.offset = %u,\n"
11161                                 "\t\t\t},\n",
11162                                 instr->meter.color_in.struct_id,
11163                                 instr->meter.color_in.n_bits,
11164                                 instr->meter.color_in.offset);
11165                 else
11166                         fprintf(f,
11167                                 "\t\t\t.color_in_val = %u,\n",
11168                                 (uint32_t)instr->meter.color_in_val);
11169
11170                 /* instr.meter.color_out. */
11171                 fprintf(f,
11172                         "\t\t\t.color_out = {\n"
11173                         "\t\t\t\t.struct_id = %u,\n"
11174                         "\t\t\t\t.n_bits = %u,\n"
11175                         "\t\t\t\t.offset = %u,\n"
11176                         "\t\t\t},\n",
11177                         instr->meter.color_out.struct_id,
11178                         instr->meter.color_out.n_bits,
11179                         instr->meter.color_out.offset);
11180         }
11181
11182         /* instr.meter and instr - closing curly braces. */
11183         fprintf(f,
11184                 "\t\t},\n"
11185                 "\t},\n");
11186 }
11187
11188 static void
11189 instr_table_export(struct instruction *instr,
11190                 FILE *f)
11191 {
11192         fprintf(f,
11193                 "\t{\n"
11194                 "\t\t.type = %s,\n"
11195                 "\t\t.table = {\n"
11196                 "\t\t\t.table_id = %u,\n"
11197                 "\t\t},\n"
11198                 "\t},\n",
11199                 instr_type_to_name(instr),
11200                 instr->table.table_id);
11201 }
11202
11203 static void
11204 instr_learn_export(struct instruction *instr, FILE *f)
11205 {
11206         fprintf(f,
11207                 "\t{\n"
11208                 "\t\t.type = %s,\n"
11209                 "\t\t.learn = {\n"
11210                 "\t\t\t\t.action_id = %u,\n"
11211                 "\t\t},\n"
11212                 "\t},\n",
11213                 instr_type_to_name(instr),
11214                 instr->learn.action_id);
11215 }
11216
11217 static void
11218 instr_forget_export(struct instruction *instr, FILE *f)
11219 {
11220         fprintf(f,
11221                 "\t{\n"
11222                 "\t\t.type = %s,\n"
11223                 "\t},\n",
11224                 instr_type_to_name(instr));
11225 }
11226
11227 static void
11228 instr_extern_export(struct instruction *instr, FILE *f)
11229 {
11230         if (instr->type == INSTR_EXTERN_OBJ)
11231                 fprintf(f,
11232                         "\t{\n"
11233                         "\t\t.type = %s,\n"
11234                         "\t\t.ext_obj = {\n"
11235                         "\t\t\t.ext_obj_id = %u,\n"
11236                         "\t\t\t.func_id = %u,\n"
11237                         "\t\t},\n"
11238                         "\t},\n",
11239                         instr_type_to_name(instr),
11240                         instr->ext_obj.ext_obj_id,
11241                         instr->ext_obj.func_id);
11242         else
11243                 fprintf(f,
11244                         "\t{\n"
11245                         "\t\t.type = %s,\n"
11246                         "\t\t.ext_func = {\n"
11247                         "\t\t\t.ext_func_id = %u,\n"
11248                         "\t\t},\n"
11249                         "\t},\n",
11250                         instr_type_to_name(instr),
11251                         instr->ext_func.ext_func_id);
11252 }
11253
11254 static void
11255 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
11256 {
11257         fprintf(f,
11258                 "\t{\n"
11259                 "\t\t.type = %s,\n"
11260                 "\t\t.jmp = {\n"
11261                 "\t\t\t.ip = NULL,\n",
11262                 instr_type_to_name(instr));
11263
11264         switch (instr->type) {
11265         case INSTR_JMP_VALID:
11266         case INSTR_JMP_INVALID:
11267                 fprintf(f,
11268                         "\t\t\t.header_id = %u,\n",
11269                         instr->jmp.header_id);
11270                 break;
11271
11272         case INSTR_JMP_ACTION_HIT:
11273         case INSTR_JMP_ACTION_MISS:
11274                 fprintf(f,
11275                         "\t\t\t.action_id = %u,\n",
11276                         instr->jmp.action_id);
11277                 break;
11278
11279         case INSTR_JMP_EQ:
11280         case INSTR_JMP_EQ_MH:
11281         case INSTR_JMP_EQ_HM:
11282         case INSTR_JMP_EQ_HH:
11283         case INSTR_JMP_NEQ:
11284         case INSTR_JMP_NEQ_MH:
11285         case INSTR_JMP_NEQ_HM:
11286         case INSTR_JMP_NEQ_HH:
11287         case INSTR_JMP_LT:
11288         case INSTR_JMP_LT_MH:
11289         case INSTR_JMP_LT_HM:
11290         case INSTR_JMP_LT_HH:
11291         case INSTR_JMP_GT:
11292         case INSTR_JMP_GT_MH:
11293         case INSTR_JMP_GT_HM:
11294         case INSTR_JMP_GT_HH:
11295                 fprintf(f,
11296                         "\t\t\t.a = {\n"
11297                         "\t\t\t\t.struct_id = %u,\n"
11298                         "\t\t\t\t.n_bits = %u,\n"
11299                         "\t\t\t\t.offset = %u,\n"
11300                         "\t\t\t},\n"
11301                         "\t\t\t.b = {\n"
11302                         "\t\t\t\t.struct_id = %u,\n"
11303                         "\t\t\t\t.n_bits = %u,\n"
11304                         "\t\t\t\t.offset = %u,\n"
11305                         "\t\t\t},\n",
11306                         instr->jmp.a.struct_id,
11307                         instr->jmp.a.n_bits,
11308                         instr->jmp.a.offset,
11309                         instr->jmp.b.struct_id,
11310                         instr->jmp.b.n_bits,
11311                         instr->jmp.b.offset);
11312                 break;
11313
11314         case INSTR_JMP_EQ_I:
11315         case INSTR_JMP_NEQ_I:
11316         case INSTR_JMP_LT_MI:
11317         case INSTR_JMP_LT_HI:
11318         case INSTR_JMP_GT_MI:
11319         case INSTR_JMP_GT_HI:
11320                 fprintf(f,
11321                         "\t\t\t.a = {\n"
11322                         "\t\t\t\t.struct_id = %u,\n"
11323                         "\t\t\t\t.n_bits = %u,\n"
11324                         "\t\t\t\t.offset = %u,\n"
11325                         "\t\t\t}\n,"
11326                         "\t\t\t.b_val = %" PRIu64 ",\n",
11327                         instr->jmp.a.struct_id,
11328                         instr->jmp.a.n_bits,
11329                         instr->jmp.a.offset,
11330                         instr->jmp.b_val);
11331                 break;
11332
11333         default:
11334                 break;
11335         }
11336
11337         fprintf(f,
11338                 "\t\t},\n"
11339                 "\t},\n");
11340 }
11341
11342 static void
11343 instr_return_export(struct instruction *instr,
11344                 FILE *f)
11345 {
11346         fprintf(f,
11347                 "\t{\n"
11348                 "\t\t.type = %s,\n",
11349                 instr_type_to_name(instr));
11350
11351         fprintf(f,
11352                 "\t},\n");
11353 }
11354
11355 static instruction_export_t export_table[] = {
11356         [INSTR_RX] = instr_io_export,
11357
11358         [INSTR_TX] = instr_io_export,
11359         [INSTR_TX_I] = instr_io_export,
11360         [INSTR_DROP] = instr_io_export,
11361         [INSTR_MIRROR] = instr_mirror_export,
11362         [INSTR_RECIRCULATE] = instr_recirculate_export,
11363         [INSTR_RECIRCID] = instr_recircid_export,
11364
11365         [INSTR_HDR_EXTRACT] = instr_io_export,
11366         [INSTR_HDR_EXTRACT2] = instr_io_export,
11367         [INSTR_HDR_EXTRACT3] = instr_io_export,
11368         [INSTR_HDR_EXTRACT4] = instr_io_export,
11369         [INSTR_HDR_EXTRACT5] = instr_io_export,
11370         [INSTR_HDR_EXTRACT6] = instr_io_export,
11371         [INSTR_HDR_EXTRACT7] = instr_io_export,
11372         [INSTR_HDR_EXTRACT8] = instr_io_export,
11373
11374         [INSTR_HDR_EXTRACT_M] = instr_io_export,
11375
11376         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
11377
11378         [INSTR_HDR_EMIT] = instr_io_export,
11379         [INSTR_HDR_EMIT_TX] = instr_io_export,
11380         [INSTR_HDR_EMIT2_TX] = instr_io_export,
11381         [INSTR_HDR_EMIT3_TX] = instr_io_export,
11382         [INSTR_HDR_EMIT4_TX] = instr_io_export,
11383         [INSTR_HDR_EMIT5_TX] = instr_io_export,
11384         [INSTR_HDR_EMIT6_TX] = instr_io_export,
11385         [INSTR_HDR_EMIT7_TX] = instr_io_export,
11386         [INSTR_HDR_EMIT8_TX] = instr_io_export,
11387
11388         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
11389         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
11390
11391         [INSTR_MOV] = instr_mov_export,
11392         [INSTR_MOV_MH] = instr_mov_export,
11393         [INSTR_MOV_HM] = instr_mov_export,
11394         [INSTR_MOV_HH] = instr_mov_export,
11395         [INSTR_MOV_I] = instr_mov_export,
11396
11397         [INSTR_DMA_HT]  = instr_dma_ht_export,
11398         [INSTR_DMA_HT2] = instr_dma_ht_export,
11399         [INSTR_DMA_HT3] = instr_dma_ht_export,
11400         [INSTR_DMA_HT4] = instr_dma_ht_export,
11401         [INSTR_DMA_HT5] = instr_dma_ht_export,
11402         [INSTR_DMA_HT6] = instr_dma_ht_export,
11403         [INSTR_DMA_HT7] = instr_dma_ht_export,
11404         [INSTR_DMA_HT8] = instr_dma_ht_export,
11405
11406         [INSTR_ALU_ADD] = instr_alu_export,
11407         [INSTR_ALU_ADD_MH] = instr_alu_export,
11408         [INSTR_ALU_ADD_HM] = instr_alu_export,
11409         [INSTR_ALU_ADD_HH] = instr_alu_export,
11410         [INSTR_ALU_ADD_MI] = instr_alu_export,
11411         [INSTR_ALU_ADD_HI] = instr_alu_export,
11412
11413         [INSTR_ALU_SUB] = instr_alu_export,
11414         [INSTR_ALU_SUB_MH] = instr_alu_export,
11415         [INSTR_ALU_SUB_HM] = instr_alu_export,
11416         [INSTR_ALU_SUB_HH] = instr_alu_export,
11417         [INSTR_ALU_SUB_MI] = instr_alu_export,
11418         [INSTR_ALU_SUB_HI] = instr_alu_export,
11419
11420         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
11421         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
11422         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
11423         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
11424
11425         [INSTR_ALU_AND] = instr_alu_export,
11426         [INSTR_ALU_AND_MH] = instr_alu_export,
11427         [INSTR_ALU_AND_HM] = instr_alu_export,
11428         [INSTR_ALU_AND_HH] = instr_alu_export,
11429         [INSTR_ALU_AND_I] = instr_alu_export,
11430
11431         [INSTR_ALU_OR] = instr_alu_export,
11432         [INSTR_ALU_OR_MH] = instr_alu_export,
11433         [INSTR_ALU_OR_HM] = instr_alu_export,
11434         [INSTR_ALU_OR_HH] = instr_alu_export,
11435         [INSTR_ALU_OR_I] = instr_alu_export,
11436
11437         [INSTR_ALU_XOR] = instr_alu_export,
11438         [INSTR_ALU_XOR_MH] = instr_alu_export,
11439         [INSTR_ALU_XOR_HM] = instr_alu_export,
11440         [INSTR_ALU_XOR_HH] = instr_alu_export,
11441         [INSTR_ALU_XOR_I] = instr_alu_export,
11442
11443         [INSTR_ALU_SHL] = instr_alu_export,
11444         [INSTR_ALU_SHL_MH] = instr_alu_export,
11445         [INSTR_ALU_SHL_HM] = instr_alu_export,
11446         [INSTR_ALU_SHL_HH] = instr_alu_export,
11447         [INSTR_ALU_SHL_MI] = instr_alu_export,
11448         [INSTR_ALU_SHL_HI] = instr_alu_export,
11449
11450         [INSTR_ALU_SHR] = instr_alu_export,
11451         [INSTR_ALU_SHR_MH] = instr_alu_export,
11452         [INSTR_ALU_SHR_HM] = instr_alu_export,
11453         [INSTR_ALU_SHR_HH] = instr_alu_export,
11454         [INSTR_ALU_SHR_MI] = instr_alu_export,
11455         [INSTR_ALU_SHR_HI] = instr_alu_export,
11456
11457         [INSTR_REGPREFETCH_RH] = instr_reg_export,
11458         [INSTR_REGPREFETCH_RM] = instr_reg_export,
11459         [INSTR_REGPREFETCH_RI] = instr_reg_export,
11460
11461         [INSTR_REGRD_HRH] = instr_reg_export,
11462         [INSTR_REGRD_HRM] = instr_reg_export,
11463         [INSTR_REGRD_MRH] = instr_reg_export,
11464         [INSTR_REGRD_MRM] = instr_reg_export,
11465         [INSTR_REGRD_HRI] = instr_reg_export,
11466         [INSTR_REGRD_MRI] = instr_reg_export,
11467
11468         [INSTR_REGWR_RHH] = instr_reg_export,
11469         [INSTR_REGWR_RHM] = instr_reg_export,
11470         [INSTR_REGWR_RMH] = instr_reg_export,
11471         [INSTR_REGWR_RMM] = instr_reg_export,
11472         [INSTR_REGWR_RHI] = instr_reg_export,
11473         [INSTR_REGWR_RMI] = instr_reg_export,
11474         [INSTR_REGWR_RIH] = instr_reg_export,
11475         [INSTR_REGWR_RIM] = instr_reg_export,
11476         [INSTR_REGWR_RII] = instr_reg_export,
11477
11478         [INSTR_REGADD_RHH] = instr_reg_export,
11479         [INSTR_REGADD_RHM] = instr_reg_export,
11480         [INSTR_REGADD_RMH] = instr_reg_export,
11481         [INSTR_REGADD_RMM] = instr_reg_export,
11482         [INSTR_REGADD_RHI] = instr_reg_export,
11483         [INSTR_REGADD_RMI] = instr_reg_export,
11484         [INSTR_REGADD_RIH] = instr_reg_export,
11485         [INSTR_REGADD_RIM] = instr_reg_export,
11486         [INSTR_REGADD_RII] = instr_reg_export,
11487
11488         [INSTR_METPREFETCH_H] = instr_meter_export,
11489         [INSTR_METPREFETCH_M] = instr_meter_export,
11490         [INSTR_METPREFETCH_I] = instr_meter_export,
11491
11492         [INSTR_METER_HHM] = instr_meter_export,
11493         [INSTR_METER_HHI] = instr_meter_export,
11494         [INSTR_METER_HMM] = instr_meter_export,
11495         [INSTR_METER_HMI] = instr_meter_export,
11496         [INSTR_METER_MHM] = instr_meter_export,
11497         [INSTR_METER_MHI] = instr_meter_export,
11498         [INSTR_METER_MMM] = instr_meter_export,
11499         [INSTR_METER_MMI] = instr_meter_export,
11500         [INSTR_METER_IHM] = instr_meter_export,
11501         [INSTR_METER_IHI] = instr_meter_export,
11502         [INSTR_METER_IMM] = instr_meter_export,
11503         [INSTR_METER_IMI] = instr_meter_export,
11504
11505         [INSTR_TABLE] = instr_table_export,
11506         [INSTR_TABLE_AF] = instr_table_export,
11507         [INSTR_SELECTOR] = instr_table_export,
11508         [INSTR_LEARNER] = instr_table_export,
11509         [INSTR_LEARNER_AF] = instr_table_export,
11510
11511         [INSTR_LEARNER_LEARN] = instr_learn_export,
11512         [INSTR_LEARNER_FORGET] = instr_forget_export,
11513
11514         [INSTR_EXTERN_OBJ] = instr_extern_export,
11515         [INSTR_EXTERN_FUNC] = instr_extern_export,
11516
11517         [INSTR_JMP] = instr_jmp_export,
11518         [INSTR_JMP_VALID] = instr_jmp_export,
11519         [INSTR_JMP_INVALID] = instr_jmp_export,
11520         [INSTR_JMP_HIT] = instr_jmp_export,
11521         [INSTR_JMP_MISS] = instr_jmp_export,
11522         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11523         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11524
11525         [INSTR_JMP_EQ] = instr_jmp_export,
11526         [INSTR_JMP_EQ_MH] = instr_jmp_export,
11527         [INSTR_JMP_EQ_HM] = instr_jmp_export,
11528         [INSTR_JMP_EQ_HH] = instr_jmp_export,
11529         [INSTR_JMP_EQ_I] = instr_jmp_export,
11530
11531         [INSTR_JMP_NEQ] = instr_jmp_export,
11532         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11533         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11534         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11535         [INSTR_JMP_NEQ_I] = instr_jmp_export,
11536
11537         [INSTR_JMP_LT] = instr_jmp_export,
11538         [INSTR_JMP_LT_MH] = instr_jmp_export,
11539         [INSTR_JMP_LT_HM] = instr_jmp_export,
11540         [INSTR_JMP_LT_HH] = instr_jmp_export,
11541         [INSTR_JMP_LT_MI] = instr_jmp_export,
11542         [INSTR_JMP_LT_HI] = instr_jmp_export,
11543
11544         [INSTR_JMP_GT] = instr_jmp_export,
11545         [INSTR_JMP_GT_MH] = instr_jmp_export,
11546         [INSTR_JMP_GT_HM] = instr_jmp_export,
11547         [INSTR_JMP_GT_HH] = instr_jmp_export,
11548         [INSTR_JMP_GT_MI] = instr_jmp_export,
11549         [INSTR_JMP_GT_HI] = instr_jmp_export,
11550
11551         [INSTR_RETURN] = instr_return_export,
11552 };
11553
11554 static void
11555 action_data_codegen(struct action *a, FILE *f)
11556 {
11557         uint32_t i;
11558
11559         fprintf(f,
11560                 "static const struct instruction action_%s_instructions[] = {\n",
11561                 a->name);
11562
11563         for (i = 0; i < a->n_instructions; i++) {
11564                 struct instruction *instr = &a->instructions[i];
11565                 instruction_export_t func = export_table[instr->type];
11566
11567                 func(instr, f);
11568         }
11569
11570         fprintf(f, "};\n");
11571 }
11572
11573 static const char *
11574 instr_type_to_func(struct instruction *instr)
11575 {
11576         switch (instr->type) {
11577         case INSTR_RX: return NULL;
11578
11579         case INSTR_TX: return "__instr_tx_exec";
11580         case INSTR_TX_I: return "__instr_tx_i_exec";
11581         case INSTR_DROP: return "__instr_drop_exec";
11582         case INSTR_MIRROR: return "__instr_mirror_exec";
11583         case INSTR_RECIRCULATE: return "__instr_recirculate_exec";
11584         case INSTR_RECIRCID: return "__instr_recircid_exec";
11585
11586         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11587         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11588         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11589         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11590         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11591         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11592         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11593         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11594
11595         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11596
11597         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11598
11599         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11600         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11601         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11602         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11603         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11604         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11605         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11606         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11607         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11608
11609         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11610         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11611
11612         case INSTR_MOV: return "__instr_mov_exec";
11613         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11614         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11615         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11616         case INSTR_MOV_I: return "__instr_mov_i_exec";
11617
11618         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11619         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11620         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11621         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11622         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11623         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11624         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11625         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11626
11627         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11628         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11629         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11630         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11631         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11632         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11633
11634         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11635         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11636         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11637         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11638         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11639         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11640
11641         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11642         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11643         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11644         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11645
11646         case INSTR_ALU_AND: return "__instr_alu_and_exec";
11647         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11648         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11649         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11650         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11651
11652         case INSTR_ALU_OR: return "__instr_alu_or_exec";
11653         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11654         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11655         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11656         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11657
11658         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11659         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11660         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11661         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11662         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11663
11664         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11665         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11666         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11667         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11668         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11669         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11670
11671         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11672         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11673         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11674         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11675         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11676         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11677
11678         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11679         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11680         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11681
11682         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11683         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11684         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11685         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11686         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11687         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11688
11689         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11690         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11691         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11692         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11693         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11694         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11695         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11696         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11697         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11698
11699         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11700         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11701         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11702         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11703         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11704         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11705         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11706         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11707         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11708
11709         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11710         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11711         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11712
11713         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11714         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11715         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11716         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11717         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11718         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11719         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11720         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11721         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11722         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11723         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11724         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11725
11726         case INSTR_TABLE: return NULL;
11727         case INSTR_TABLE_AF: return NULL;
11728         case INSTR_SELECTOR: return NULL;
11729         case INSTR_LEARNER: return NULL;
11730         case INSTR_LEARNER_AF: return NULL;
11731
11732         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11733         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11734
11735         case INSTR_EXTERN_OBJ: return NULL;
11736         case INSTR_EXTERN_FUNC: return NULL;
11737
11738         case INSTR_JMP: return NULL;
11739         case INSTR_JMP_VALID: return NULL;
11740         case INSTR_JMP_INVALID: return NULL;
11741         case INSTR_JMP_HIT: return NULL;
11742         case INSTR_JMP_MISS: return NULL;
11743         case INSTR_JMP_ACTION_HIT: return NULL;
11744         case INSTR_JMP_ACTION_MISS: return NULL;
11745         case INSTR_JMP_EQ: return NULL;
11746         case INSTR_JMP_EQ_MH: return NULL;
11747         case INSTR_JMP_EQ_HM: return NULL;
11748         case INSTR_JMP_EQ_HH: return NULL;
11749         case INSTR_JMP_EQ_I: return NULL;
11750         case INSTR_JMP_NEQ: return NULL;
11751         case INSTR_JMP_NEQ_MH: return NULL;
11752         case INSTR_JMP_NEQ_HM: return NULL;
11753         case INSTR_JMP_NEQ_HH: return NULL;
11754         case INSTR_JMP_NEQ_I: return NULL;
11755         case INSTR_JMP_LT: return NULL;
11756         case INSTR_JMP_LT_MH: return NULL;
11757         case INSTR_JMP_LT_HM: return NULL;
11758         case INSTR_JMP_LT_HH: return NULL;
11759         case INSTR_JMP_LT_MI: return NULL;
11760         case INSTR_JMP_LT_HI: return NULL;
11761         case INSTR_JMP_GT: return NULL;
11762         case INSTR_JMP_GT_MH: return NULL;
11763         case INSTR_JMP_GT_HM: return NULL;
11764         case INSTR_JMP_GT_HH: return NULL;
11765         case INSTR_JMP_GT_MI: return NULL;
11766         case INSTR_JMP_GT_HI: return NULL;
11767
11768         case INSTR_RETURN: return NULL;
11769
11770         default: return NULL;
11771         }
11772 }
11773
11774 static void
11775 action_instr_does_tx_codegen(struct action *a,
11776                         uint32_t instr_pos,
11777                         struct instruction *instr,
11778                         FILE *f)
11779 {
11780         fprintf(f,
11781                 "%s(p, t, &action_%s_instructions[%u]);\n"
11782                 "\tthread_ip_reset(p, t);\n"
11783                 "\tinstr_rx_exec(p);\n"
11784                 "\treturn;\n",
11785                 instr_type_to_func(instr),
11786                 a->name,
11787                 instr_pos);
11788 }
11789
11790 static void
11791 action_instr_extern_obj_codegen(struct action *a,
11792                                 uint32_t instr_pos,
11793                                 FILE *f)
11794 {
11795         fprintf(f,
11796                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11797                 a->name,
11798                 instr_pos);
11799 }
11800
11801 static void
11802 action_instr_extern_func_codegen(struct action *a,
11803                                  uint32_t instr_pos,
11804                                  FILE *f)
11805 {
11806         fprintf(f,
11807                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11808                 a->name,
11809                 instr_pos);
11810 }
11811
11812 static void
11813 action_instr_jmp_codegen(struct action *a,
11814                          uint32_t instr_pos,
11815                          struct instruction *instr,
11816                          struct instruction_data *data,
11817                          FILE *f)
11818 {
11819         switch (instr->type) {
11820         case INSTR_JMP:
11821                 fprintf(f,
11822                         "goto %s;\n",
11823                         data->jmp_label);
11824                 return;
11825
11826         case INSTR_JMP_VALID:
11827                 fprintf(f,
11828                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11829                         "\t\tgoto %s;\n",
11830                         a->name,
11831                         instr_pos,
11832                         data->jmp_label);
11833                 return;
11834
11835         case INSTR_JMP_INVALID:
11836                 fprintf(f,
11837                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11838                         "\t\tgoto %s;\n",
11839                         a->name,
11840                         instr_pos,
11841                         data->jmp_label);
11842                 return;
11843
11844         case INSTR_JMP_HIT:
11845                 fprintf(f,
11846                         "if (t->hit)\n"
11847                         "\t\tgoto %s;\n",
11848                         data->jmp_label);
11849                 return;
11850
11851         case INSTR_JMP_MISS:
11852                 fprintf(f,
11853                         "if (!t->hit)\n"
11854                         "\t\tgoto %s;\n",
11855                         data->jmp_label);
11856                 return;
11857
11858         case INSTR_JMP_ACTION_HIT:
11859                 fprintf(f,
11860                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11861                         "\t\tgoto %s;\n",
11862                         a->name,
11863                         instr_pos,
11864                         data->jmp_label);
11865                 return;
11866
11867         case INSTR_JMP_ACTION_MISS:
11868                 fprintf(f,
11869                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11870                         "\t\tgoto %s;\n",
11871                         a->name,
11872                         instr_pos,
11873                         data->jmp_label);
11874                 return;
11875
11876         case INSTR_JMP_EQ:
11877                 fprintf(f,
11878                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11879                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11880                         "\t\tgoto %s;\n",
11881                         a->name,
11882                         instr_pos,
11883                         a->name,
11884                         instr_pos,
11885                         data->jmp_label);
11886                 return;
11887
11888         case INSTR_JMP_EQ_MH:
11889                 fprintf(f,
11890                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11891                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11892                         "\t\tgoto %s;\n",
11893                         a->name,
11894                         instr_pos,
11895                         a->name,
11896                         instr_pos,
11897                         data->jmp_label);
11898                 return;
11899
11900         case INSTR_JMP_EQ_HM:
11901                 fprintf(f,
11902                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11903                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11904                         "\t\tgoto %s;\n",
11905                         a->name,
11906                         instr_pos,
11907                         a->name,
11908                         instr_pos,
11909                         data->jmp_label);
11910                 return;
11911
11912         case INSTR_JMP_EQ_HH:
11913                 fprintf(f,
11914                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11915                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11916                         "\t\tgoto %s;\n",
11917                         a->name,
11918                         instr_pos,
11919                         a->name,
11920                         instr_pos,
11921                         data->jmp_label);
11922                 return;
11923
11924         case INSTR_JMP_EQ_I:
11925                 fprintf(f,
11926                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11927                         "action_%s_instructions[%u].jmp.b_val)\n"
11928                         "\t\tgoto %s;\n",
11929                         a->name,
11930                         instr_pos,
11931                         a->name,
11932                         instr_pos,
11933                         data->jmp_label);
11934                 return;
11935
11936         case INSTR_JMP_NEQ:
11937                 fprintf(f,
11938                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11939                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11940                         "\t\tgoto %s;\n",
11941                         a->name,
11942                         instr_pos,
11943                         a->name,
11944                         instr_pos,
11945                         data->jmp_label);
11946                 return;
11947
11948         case INSTR_JMP_NEQ_MH:
11949                 fprintf(f,
11950                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11951                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11952                         "\t\tgoto %s;\n",
11953                         a->name,
11954                         instr_pos,
11955                         a->name,
11956                         instr_pos,
11957                         data->jmp_label);
11958                 return;
11959
11960         case INSTR_JMP_NEQ_HM:
11961                 fprintf(f,
11962                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11963                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11964                         "\t\tgoto %s;\n",
11965                         a->name,
11966                         instr_pos,
11967                         a->name,
11968                         instr_pos,
11969                         data->jmp_label);
11970                 return;
11971
11972         case INSTR_JMP_NEQ_HH:
11973                 fprintf(f,
11974                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11975                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11976                         "\t\tgoto %s;\n",
11977                         a->name,
11978                         instr_pos,
11979                         a->name,
11980                         instr_pos,
11981                         data->jmp_label);
11982                 return;
11983
11984         case INSTR_JMP_NEQ_I:
11985                 fprintf(f,
11986                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11987                         "action_%s_instructions[%u].jmp.b_val)\n"
11988                         "\t\tgoto %s;\n",
11989                         a->name,
11990                         instr_pos,
11991                         a->name,
11992                         instr_pos,
11993                         data->jmp_label);
11994                 return;
11995
11996         case INSTR_JMP_LT:
11997                 fprintf(f,
11998                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11999                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12000                         "\t\tgoto %s;\n",
12001                         a->name,
12002                         instr_pos,
12003                         a->name,
12004                         instr_pos,
12005                         data->jmp_label);
12006                 return;
12007
12008         case INSTR_JMP_LT_MH:
12009                 fprintf(f,
12010                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12011                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12012                         "\t\tgoto %s;\n",
12013                         a->name,
12014                         instr_pos,
12015                         a->name,
12016                         instr_pos,
12017                         data->jmp_label);
12018                 return;
12019
12020         case INSTR_JMP_LT_HM:
12021                 fprintf(f,
12022                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12023                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12024                         "\t\tgoto %s;\n",
12025                         a->name,
12026                         instr_pos,
12027                         a->name,
12028                         instr_pos,
12029                         data->jmp_label);
12030                 return;
12031
12032         case INSTR_JMP_LT_HH:
12033                 fprintf(f,
12034                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12035                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12036                         "\t\tgoto %s;\n",
12037                         a->name,
12038                         instr_pos,
12039                         a->name,
12040                         instr_pos,
12041                         data->jmp_label);
12042                 return;
12043
12044         case INSTR_JMP_LT_MI:
12045                 fprintf(f,
12046                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12047                         "action_%s_instructions[%u].jmp.b_val)\n"
12048                         "\t\tgoto %s;\n",
12049                         a->name,
12050                         instr_pos,
12051                         a->name,
12052                         instr_pos,
12053                         data->jmp_label);
12054                 return;
12055
12056         case INSTR_JMP_LT_HI:
12057                 fprintf(f,
12058                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12059                         "action_%s_instructions[%u].jmp.b_val)\n"
12060                         "\t\tgoto %s;\n",
12061                         a->name,
12062                         instr_pos,
12063                         a->name,
12064                         instr_pos,
12065                         data->jmp_label);
12066                 return;
12067
12068         case INSTR_JMP_GT:
12069                 fprintf(f,
12070                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12071                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12072                         "\t\tgoto %s;\n",
12073                         a->name,
12074                         instr_pos,
12075                         a->name,
12076                         instr_pos,
12077                         data->jmp_label);
12078                 return;
12079
12080         case INSTR_JMP_GT_MH:
12081                 fprintf(f,
12082                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12083                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12084                         "\t\tgoto %s;\n",
12085                         a->name,
12086                         instr_pos,
12087                         a->name,
12088                         instr_pos,
12089                         data->jmp_label);
12090                 return;
12091
12092         case INSTR_JMP_GT_HM:
12093                 fprintf(f,
12094                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12095                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12096                         "\t\tgoto %s;\n",
12097                         a->name,
12098                         instr_pos,
12099                         a->name,
12100                         instr_pos,
12101                         data->jmp_label);
12102                 return;
12103
12104         case INSTR_JMP_GT_HH:
12105                 fprintf(f,
12106                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12107                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12108                         "\t\tgoto %s;\n",
12109                         a->name,
12110                         instr_pos,
12111                         a->name,
12112                         instr_pos,
12113                         data->jmp_label);
12114                 return;
12115
12116         case INSTR_JMP_GT_MI:
12117                 fprintf(f,
12118                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12119                         "action_%s_instructions[%u].jmp.b_val)\n"
12120                         "\t\tgoto %s;\n",
12121                         a->name,
12122                         instr_pos,
12123                         a->name,
12124                         instr_pos,
12125                         data->jmp_label);
12126                 return;
12127
12128         case INSTR_JMP_GT_HI:
12129                 fprintf(f,
12130                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12131                         "action_%s_instructions[%u].jmp.b_val)\n"
12132                         "\t\tgoto %s;\n",
12133                         a->name,
12134                         instr_pos,
12135                         a->name,
12136                         instr_pos,
12137                         data->jmp_label);
12138                 return;
12139
12140         default:
12141                 return;
12142         }
12143 }
12144
12145 static void
12146 action_instr_return_codegen(FILE *f)
12147 {
12148         fprintf(f,
12149                 "return;\n");
12150 }
12151
12152 static void
12153 action_instr_codegen(struct action *a, FILE *f)
12154 {
12155         uint32_t i;
12156
12157         fprintf(f,
12158                 "void\n"
12159                 "action_%s_run(struct rte_swx_pipeline *p)\n"
12160                 "{\n"
12161                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12162                 "\n",
12163                 a->name);
12164
12165         for (i = 0; i < a->n_instructions; i++) {
12166                 struct instruction *instr = &a->instructions[i];
12167                 struct instruction_data *data = &a->instruction_data[i];
12168
12169                 /* Label, if present. */
12170                 if (data->label[0])
12171                         fprintf(f, "\n%s : ", data->label);
12172                 else
12173                         fprintf(f, "\n\t");
12174
12175                 /* TX instruction type. */
12176                 if (instruction_does_tx(instr)) {
12177                         action_instr_does_tx_codegen(a, i, instr, f);
12178                         continue;
12179                 }
12180
12181                 /* Extern object/function instruction type. */
12182                 if (instr->type == INSTR_EXTERN_OBJ) {
12183                         action_instr_extern_obj_codegen(a, i, f);
12184                         continue;
12185                 }
12186
12187                 if (instr->type == INSTR_EXTERN_FUNC) {
12188                         action_instr_extern_func_codegen(a, i, f);
12189                         continue;
12190                 }
12191
12192                 /* Jump instruction type. */
12193                 if (instruction_is_jmp(instr)) {
12194                         action_instr_jmp_codegen(a, i, instr, data, f);
12195                         continue;
12196                 }
12197
12198                 /* Return instruction type. */
12199                 if (instr->type == INSTR_RETURN) {
12200                         action_instr_return_codegen(f);
12201                         continue;
12202                 }
12203
12204                 /* Any other instruction type. */
12205                 fprintf(f,
12206                         "%s(p, t, &action_%s_instructions[%u]);\n",
12207                         instr_type_to_func(instr),
12208                         a->name,
12209                         i);
12210         }
12211
12212         fprintf(f, "}\n\n");
12213 }
12214
12215 struct instruction_group {
12216         TAILQ_ENTRY(instruction_group) node;
12217
12218         uint32_t group_id;
12219
12220         uint32_t first_instr_id;
12221
12222         uint32_t last_instr_id;
12223
12224         instr_exec_t func;
12225 };
12226
12227 TAILQ_HEAD(instruction_group_list, instruction_group);
12228
12229 static struct instruction_group *
12230 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
12231 {
12232         struct instruction_group *g;
12233
12234         TAILQ_FOREACH(g, igl, node)
12235                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
12236                         return g;
12237
12238         return NULL;
12239 }
12240
12241 static void
12242 instruction_group_list_free(struct instruction_group_list *igl)
12243 {
12244         if (!igl)
12245                 return;
12246
12247         for ( ; ; ) {
12248                 struct instruction_group *g;
12249
12250                 g = TAILQ_FIRST(igl);
12251                 if (!g)
12252                         break;
12253
12254                 TAILQ_REMOVE(igl, g, node);
12255                 free(g);
12256         }
12257
12258         free(igl);
12259 }
12260
12261 static struct instruction_group_list *
12262 instruction_group_list_create(struct rte_swx_pipeline *p)
12263 {
12264         struct instruction_group_list *igl = NULL;
12265         struct instruction_group *g = NULL;
12266         uint32_t n_groups = 0, i;
12267
12268         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
12269                 goto error;
12270
12271         /* List init. */
12272         igl = calloc(1, sizeof(struct instruction_group_list));
12273         if (!igl)
12274                 goto error;
12275
12276         TAILQ_INIT(igl);
12277
12278         /* Allocate the first group. */
12279         g = calloc(1, sizeof(struct instruction_group));
12280         if (!g)
12281                 goto error;
12282
12283         /* Iteration 1: Separate the instructions into groups based on the thread yield
12284          * instructions. Do not worry about the jump instructions at this point.
12285          */
12286         for (i = 0; i < p->n_instructions; i++) {
12287                 struct instruction *instr = &p->instructions[i];
12288
12289                 /* Check for thread yield instructions. */
12290                 if (!instruction_does_thread_yield(instr))
12291                         continue;
12292
12293                 /* If the current group contains at least one instruction, then finalize it (with
12294                  * the previous instruction), add it to the list and allocate a new group (that
12295                  * starts with the current instruction).
12296                  */
12297                 if (i - g->first_instr_id) {
12298                         /* Finalize the group. */
12299                         g->last_instr_id = i - 1;
12300
12301                         /* Add the group to the list. Advance the number of groups. */
12302                         TAILQ_INSERT_TAIL(igl, g, node);
12303                         n_groups++;
12304
12305                         /* Allocate a new group. */
12306                         g = calloc(1, sizeof(struct instruction_group));
12307                         if (!g)
12308                                 goto error;
12309
12310                         /* Initialize the new group. */
12311                         g->group_id = n_groups;
12312                         g->first_instr_id = i;
12313                 }
12314
12315                 /* Finalize the current group (with the current instruction, therefore this group
12316                  * contains just the current thread yield instruction), add it to the list and
12317                  * allocate a new group (that starts with the next instruction).
12318                  */
12319
12320                 /* Finalize the group. */
12321                 g->last_instr_id = i;
12322
12323                 /* Add the group to the list. Advance the number of groups. */
12324                 TAILQ_INSERT_TAIL(igl, g, node);
12325                 n_groups++;
12326
12327                 /* Allocate a new group. */
12328                 g = calloc(1, sizeof(struct instruction_group));
12329                 if (!g)
12330                         goto error;
12331
12332                 /* Initialize the new group. */
12333                 g->group_id = n_groups;
12334                 g->first_instr_id = i + 1;
12335         }
12336
12337         /* Handle the last group. */
12338         if (i - g->first_instr_id) {
12339                 /* Finalize the group. */
12340                 g->last_instr_id = i - 1;
12341
12342                 /* Add the group to the list. Advance the number of groups. */
12343                 TAILQ_INSERT_TAIL(igl, g, node);
12344                 n_groups++;
12345         } else
12346                 free(g);
12347
12348         g = NULL;
12349
12350         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
12351          * the destination of a jump instruction located in a different group ("far jump"), then the
12352          * current group has to be split, so that the instruction representing the far jump
12353          * destination is at the start of its group.
12354          */
12355         for ( ; ; ) {
12356                 int is_modified = 0;
12357
12358                 for (i = 0; i < p->n_instructions; i++) {
12359                         struct instruction_data *data = &p->instruction_data[i];
12360                         struct instruction_group *g;
12361                         uint32_t j;
12362
12363                         /* Continue when the current instruction is not a jump destination. */
12364                         if (!data->n_users)
12365                                 continue;
12366
12367                         g = instruction_group_list_group_find(igl, i);
12368                         if (!g)
12369                                 goto error;
12370
12371                         /* Find out all the jump instructions with this destination. */
12372                         for (j = 0; j < p->n_instructions; j++) {
12373                                 struct instruction *jmp_instr = &p->instructions[j];
12374                                 struct instruction_data *jmp_data = &p->instruction_data[j];
12375                                 struct instruction_group *jmp_g, *new_g;
12376
12377                                 /* Continue when not a jump instruction. Even when jump instruction,
12378                                  * continue when the jump destination is not this instruction.
12379                                  */
12380                                 if (!instruction_is_jmp(jmp_instr) ||
12381                                     strcmp(jmp_data->jmp_label, data->label))
12382                                         continue;
12383
12384                                 jmp_g = instruction_group_list_group_find(igl, j);
12385                                 if (!jmp_g)
12386                                         goto error;
12387
12388                                 /* Continue when both the jump instruction and the jump destination
12389                                  * instruction are in the same group. Even when in different groups,
12390                                  * still continue if the jump destination instruction is already the
12391                                  * first instruction of its group.
12392                                  */
12393                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
12394                                         continue;
12395
12396                                 /* Split the group of the current jump destination instruction to
12397                                  * make this instruction the first instruction of a new group.
12398                                  */
12399                                 new_g = calloc(1, sizeof(struct instruction_group));
12400                                 if (!new_g)
12401                                         goto error;
12402
12403                                 new_g->group_id = n_groups;
12404                                 new_g->first_instr_id = i;
12405                                 new_g->last_instr_id = g->last_instr_id;
12406
12407                                 g->last_instr_id = i - 1;
12408
12409                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
12410                                 n_groups++;
12411                                 is_modified = 1;
12412
12413                                 /* The decision to split this group (to make the current instruction
12414                                  * the first instruction of a new group) is already taken and fully
12415                                  * implemented, so no need to search for more reasons to do it.
12416                                  */
12417                                 break;
12418                         }
12419                 }
12420
12421                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
12422                  * previously considered local (i.e. the jump destination is in the same group as
12423                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
12424                  * different group than the jump instruction). Wost case scenario: each instruction
12425                  * that is a jump destination ends up as the first instruction of its group.
12426                  */
12427                 if (!is_modified)
12428                         break;
12429         }
12430
12431         /* Re-assign the group IDs to be in incremental order. */
12432         i = 0;
12433         TAILQ_FOREACH(g, igl, node) {
12434                 g->group_id = i;
12435
12436                 i++;
12437         }
12438
12439         return igl;
12440
12441 error:
12442         instruction_group_list_free(igl);
12443
12444         free(g);
12445
12446         return NULL;
12447 }
12448
12449 static void
12450 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12451                                uint32_t instr_pos,
12452                                struct instruction *instr,
12453                                FILE *f)
12454 {
12455         fprintf(f,
12456                 "%s(p, t, &pipeline_instructions[%u]);\n"
12457                 "\tthread_ip_reset(p, t);\n"
12458                 "\tinstr_rx_exec(p);\n"
12459                 "\treturn;\n",
12460                 instr_type_to_func(instr),
12461                 instr_pos);
12462 }
12463
12464 static int
12465 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12466                            struct instruction_group_list *igl,
12467                            uint32_t jmp_instr_id,
12468                            struct instruction *jmp_instr,
12469                            struct instruction_data *jmp_data,
12470                            FILE *f)
12471 {
12472         struct instruction_group *jmp_g, *g;
12473         struct instruction_data *data;
12474         uint32_t instr_id;
12475
12476         switch (jmp_instr->type) {
12477         case INSTR_JMP:
12478                 break;
12479
12480         case INSTR_JMP_VALID:
12481                 fprintf(f,
12482                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12483                         jmp_instr_id);
12484                 break;
12485
12486         case INSTR_JMP_INVALID:
12487                 fprintf(f,
12488                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12489                         jmp_instr_id);
12490                 break;
12491
12492         case INSTR_JMP_HIT:
12493                 fprintf(f,
12494                         "if (t->hit)\n");
12495                 break;
12496
12497         case INSTR_JMP_MISS:
12498                 fprintf(f,
12499                         "if (!t->hit)\n");
12500                 break;
12501
12502         case INSTR_JMP_ACTION_HIT:
12503                 fprintf(f,
12504                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12505                         jmp_instr_id);
12506                 break;
12507
12508         case INSTR_JMP_ACTION_MISS:
12509                 fprintf(f,
12510                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12511                         jmp_instr_id);
12512                 break;
12513
12514         case INSTR_JMP_EQ:
12515                 fprintf(f,
12516                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12517                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12518                         jmp_instr_id,
12519                         jmp_instr_id);
12520                 break;
12521
12522         case INSTR_JMP_EQ_MH:
12523                 fprintf(f,
12524                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12525                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12526                         jmp_instr_id,
12527                         jmp_instr_id);
12528                 break;
12529
12530         case INSTR_JMP_EQ_HM:
12531                 fprintf(f,
12532                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12533                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12534                         jmp_instr_id,
12535                         jmp_instr_id);
12536                 break;
12537
12538         case INSTR_JMP_EQ_HH:
12539                 fprintf(f,
12540                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12541                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12542                         jmp_instr_id,
12543                         jmp_instr_id);
12544                 break;
12545
12546         case INSTR_JMP_EQ_I:
12547                 fprintf(f,
12548                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12549                         "pipeline_instructions[%u].jmp.b_val)",
12550                         jmp_instr_id,
12551                         jmp_instr_id);
12552                 break;
12553
12554         case INSTR_JMP_NEQ:
12555                 fprintf(f,
12556                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12557                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12558                         jmp_instr_id,
12559                         jmp_instr_id);
12560                 break;
12561
12562         case INSTR_JMP_NEQ_MH:
12563                 fprintf(f,
12564                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12565                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12566                         jmp_instr_id,
12567                         jmp_instr_id);
12568                 break;
12569
12570         case INSTR_JMP_NEQ_HM:
12571                 fprintf(f,
12572                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12573                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12574                         jmp_instr_id,
12575                         jmp_instr_id);
12576                 break;
12577
12578         case INSTR_JMP_NEQ_HH:
12579                 fprintf(f,
12580                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12581                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12582                         jmp_instr_id,
12583                         jmp_instr_id);
12584                 break;
12585
12586         case INSTR_JMP_NEQ_I:
12587                 fprintf(f,
12588                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12589                         "pipeline_instructions[%u].jmp.b_val)",
12590                         jmp_instr_id,
12591                         jmp_instr_id);
12592                 break;
12593
12594         case INSTR_JMP_LT:
12595                 fprintf(f,
12596                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12597                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12598                         jmp_instr_id,
12599                         jmp_instr_id);
12600                 break;
12601
12602         case INSTR_JMP_LT_MH:
12603                 fprintf(f,
12604                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12605                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12606                         jmp_instr_id,
12607                         jmp_instr_id);
12608                 break;
12609
12610         case INSTR_JMP_LT_HM:
12611                 fprintf(f,
12612                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12613                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12614                         jmp_instr_id,
12615                         jmp_instr_id);
12616                 break;
12617
12618         case INSTR_JMP_LT_HH:
12619                 fprintf(f,
12620                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12621                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12622                         jmp_instr_id,
12623                         jmp_instr_id);
12624                 break;
12625
12626         case INSTR_JMP_LT_MI:
12627                 fprintf(f,
12628                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12629                         "pipeline_instructions[%u].jmp.b_val)",
12630                         jmp_instr_id,
12631                         jmp_instr_id);
12632                 break;
12633
12634         case INSTR_JMP_LT_HI:
12635                 fprintf(f,
12636                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12637                         "pipeline_instructions[%u].jmp.b_val)",
12638                         jmp_instr_id,
12639                         jmp_instr_id);
12640                 break;
12641
12642         case INSTR_JMP_GT:
12643                 fprintf(f,
12644                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12645                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12646                         jmp_instr_id,
12647                         jmp_instr_id);
12648                 break;
12649
12650         case INSTR_JMP_GT_MH:
12651                 fprintf(f,
12652                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12653                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12654                         jmp_instr_id,
12655                         jmp_instr_id);
12656                 break;
12657
12658         case INSTR_JMP_GT_HM:
12659                 fprintf(f,
12660                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12661                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12662                         jmp_instr_id,
12663                         jmp_instr_id);
12664                 break;
12665
12666         case INSTR_JMP_GT_HH:
12667                 fprintf(f,
12668                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12669                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12670                         jmp_instr_id,
12671                         jmp_instr_id);
12672                 break;
12673
12674         case INSTR_JMP_GT_MI:
12675                 fprintf(f,
12676                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12677                         "pipeline_instructions[%u].jmp.b_val)",
12678                         jmp_instr_id,
12679                         jmp_instr_id);
12680                 break;
12681
12682         case INSTR_JMP_GT_HI:
12683                 fprintf(f,
12684                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12685                         "pipeline_instructions[%u].jmp.b_val)",
12686                         jmp_instr_id,
12687                         jmp_instr_id);
12688                 break;
12689
12690         default:
12691                 break;
12692         }
12693
12694         /* Find the instruction group of the jump instruction. */
12695         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12696         if (!jmp_g)
12697                 return -EINVAL;
12698
12699         /* Find the instruction group of the jump destination instruction. */
12700         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12701         if (!data)
12702                 return -EINVAL;
12703
12704         instr_id = data - p->instruction_data;
12705
12706         g = instruction_group_list_group_find(igl, instr_id);
12707         if (!g)
12708                 return -EINVAL;
12709
12710         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12711          * instruction group).
12712          */
12713         if (g->group_id == jmp_g->group_id)
12714                 fprintf(f,
12715                         "\n\t\tgoto %s;\n",
12716                         jmp_data->jmp_label);
12717         else
12718                 fprintf(f,
12719                         " {\n"
12720                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12721                         "\t\treturn;\n"
12722                         "\t}\n\n",
12723                         g->group_id);
12724
12725         return 0;
12726 }
12727
12728 static void
12729 instruction_group_list_codegen(struct instruction_group_list *igl,
12730                                struct rte_swx_pipeline *p,
12731                                FILE *f)
12732 {
12733         struct instruction_group *g;
12734         uint32_t i;
12735         int is_required = 0;
12736
12737         /* Check if code generation is required. */
12738         TAILQ_FOREACH(g, igl, node)
12739                 if (g->first_instr_id < g->last_instr_id)
12740                         is_required = 1;
12741
12742         if (!is_required)
12743                 return;
12744
12745         /* Generate the code for the pipeline instruction array. */
12746         fprintf(f,
12747                 "static const struct instruction pipeline_instructions[] = {\n");
12748
12749         for (i = 0; i < p->n_instructions; i++) {
12750                 struct instruction *instr = &p->instructions[i];
12751                 instruction_export_t func = export_table[instr->type];
12752
12753                 func(instr, f);
12754         }
12755
12756         fprintf(f, "};\n\n");
12757
12758         /* Generate the code for the pipeline functions: one function for each instruction group
12759          * that contains more than one instruction.
12760          */
12761         TAILQ_FOREACH(g, igl, node) {
12762                 struct instruction *last_instr;
12763                 uint32_t j;
12764
12765                 /* Skip if group contains a single instruction. */
12766                 if (g->last_instr_id == g->first_instr_id)
12767                         continue;
12768
12769                 /* Generate new pipeline function. */
12770                 fprintf(f,
12771                         "void\n"
12772                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12773                         "{\n"
12774                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12775                         "\n",
12776                         g->group_id);
12777
12778                 /* Generate the code for each pipeline instruction. */
12779                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12780                         struct instruction *instr = &p->instructions[j];
12781                         struct instruction_data *data = &p->instruction_data[j];
12782
12783                         /* Label, if present. */
12784                         if (data->label[0])
12785                                 fprintf(f, "\n%s : ", data->label);
12786                         else
12787                                 fprintf(f, "\n\t");
12788
12789                         /* TX instruction type. */
12790                         if (instruction_does_tx(instr)) {
12791                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12792                                 continue;
12793                         }
12794
12795                         /* Jump instruction type. */
12796                         if (instruction_is_jmp(instr)) {
12797                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12798                                 continue;
12799                         }
12800
12801                         /* Any other instruction type. */
12802                         fprintf(f,
12803                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12804                                 instr_type_to_func(instr),
12805                                 j);
12806                 }
12807
12808                 /* Finalize the generated pipeline function. For some instructions such as TX,
12809                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12810                  * decided unconditionally and the instruction pointer of the current thread set
12811                  * accordingly; for all the other instructions, the instruction pointer must be
12812                  * incremented now.
12813                  */
12814                 last_instr = &p->instructions[g->last_instr_id];
12815
12816                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12817                         fprintf(f,
12818                                 "thread_ip_inc(p);\n");
12819
12820                 fprintf(f,
12821                         "}\n"
12822                         "\n");
12823         }
12824 }
12825
12826 static uint32_t
12827 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12828 {
12829         struct instruction_group *g;
12830         uint32_t n_custom_instr = 0;
12831
12832         /* Groups with a single instruction: no function is generated for this group, the group
12833          * keeps its current instruction. Groups with more than two instructions: one function and
12834          * the associated custom instruction get generated for each such group.
12835          */
12836         TAILQ_FOREACH(g, igl, node) {
12837                 if (g->first_instr_id == g->last_instr_id)
12838                         continue;
12839
12840                 n_custom_instr++;
12841         }
12842
12843         return n_custom_instr;
12844 }
12845
12846 static int
12847 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12848 {
12849         struct action *a;
12850         FILE *f = NULL;
12851
12852         /* Create the .c file. */
12853         f = fopen("/tmp/pipeline.c", "w");
12854         if (!f)
12855                 return -EIO;
12856
12857         /* Include the .h file. */
12858         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12859
12860         /* Add the code for each action. */
12861         TAILQ_FOREACH(a, &p->actions, node) {
12862                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12863
12864                 action_data_codegen(a, f);
12865
12866                 fprintf(f, "\n");
12867
12868                 action_instr_codegen(a, f);
12869
12870                 fprintf(f, "\n");
12871         }
12872
12873         /* Add the pipeline code. */
12874         instruction_group_list_codegen(igl, p, f);
12875
12876         /* Close the .c file. */
12877         fclose(f);
12878
12879         return 0;
12880 }
12881
12882 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12883 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12884 #endif
12885
12886 static int
12887 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12888 {
12889         struct action *a;
12890         struct instruction_group *g;
12891         char *dir_in, *buffer = NULL;
12892         const char *dir_out;
12893         int status = 0;
12894
12895         /* Get the environment variables. */
12896         dir_in = getenv("RTE_INSTALL_DIR");
12897         if (!dir_in) {
12898                 status = -EINVAL;
12899                 goto free;
12900         }
12901
12902         dir_out = "/tmp";
12903
12904         /* Memory allocation for the command buffer. */
12905         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12906         if (!buffer) {
12907                 status = -ENOMEM;
12908                 goto free;
12909         }
12910
12911         snprintf(buffer,
12912                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12913                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12914                  "-I %s/lib/pipeline "
12915                  "-I %s/lib/eal/include "
12916                  "-I %s/lib/eal/x86/include "
12917                  "-I %s/lib/eal/include/generic "
12918                  "-I %s/lib/meter "
12919                  "-I %s/lib/port "
12920                  "-I %s/lib/table "
12921                  "-I %s/lib/pipeline "
12922                  "-I %s/config "
12923                  "-I %s/build "
12924                  "-I %s/lib/eal/linux/include "
12925                  ">%s/pipeline.log 2>&1 "
12926                  "&& "
12927                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12928                  ">>%s/pipeline.log 2>&1",
12929                  dir_out,
12930                  dir_out,
12931                  dir_in,
12932                  dir_in,
12933                  dir_in,
12934                  dir_in,
12935                  dir_in,
12936                  dir_in,
12937                  dir_in,
12938                  dir_in,
12939                  dir_in,
12940                  dir_in,
12941                  dir_in,
12942                  dir_out,
12943                  dir_out,
12944                  dir_out,
12945                  dir_out);
12946
12947         /* Build the shared object library. */
12948         status = system(buffer);
12949         if (status)
12950                 goto free;
12951
12952         /* Open library. */
12953         snprintf(buffer,
12954                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12955                  "%s/libpipeline.so",
12956                  dir_out);
12957
12958         p->lib = dlopen(buffer, RTLD_LAZY);
12959         if (!p->lib) {
12960                 status = -EIO;
12961                 goto free;
12962         }
12963
12964         /* Get the action function symbols. */
12965         TAILQ_FOREACH(a, &p->actions, node) {
12966                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12967
12968                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12969                 if (!p->action_funcs[a->id]) {
12970                         status = -EINVAL;
12971                         goto free;
12972                 }
12973         }
12974
12975         /* Get the pipeline function symbols. */
12976         TAILQ_FOREACH(g, igl, node) {
12977                 if (g->first_instr_id == g->last_instr_id)
12978                         continue;
12979
12980                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12981
12982                 g->func = dlsym(p->lib, buffer);
12983                 if (!g->func) {
12984                         status = -EINVAL;
12985                         goto free;
12986                 }
12987         }
12988
12989 free:
12990         if (status && p->lib) {
12991                 dlclose(p->lib);
12992                 p->lib = NULL;
12993         }
12994
12995         free(buffer);
12996
12997         return status;
12998 }
12999
13000 static int
13001 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
13002                       struct instruction_group_list *igl)
13003 {
13004         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
13005
13006         /* Check that enough space is available within the pipeline instruction table to store all
13007          * the custom instructions.
13008          */
13009         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
13010                 return -ENOSPC;
13011
13012         return 0;
13013 }
13014
13015 static void
13016 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13017 {
13018         struct instruction_group *g;
13019         uint32_t i;
13020
13021         /* Pipeline table instructions. */
13022         for (i = 0; i < p->n_instructions; i++) {
13023                 struct instruction *instr = &p->instructions[i];
13024
13025                 if (instr->type == INSTR_TABLE)
13026                         instr->type = INSTR_TABLE_AF;
13027
13028                 if (instr->type == INSTR_LEARNER)
13029                         instr->type = INSTR_LEARNER_AF;
13030         }
13031
13032         /* Pipeline custom instructions. */
13033         i = 0;
13034         TAILQ_FOREACH(g, igl, node) {
13035                 struct instruction *instr = &p->instructions[g->first_instr_id];
13036                 uint32_t j;
13037
13038                 if (g->first_instr_id == g->last_instr_id)
13039                         continue;
13040
13041                 /* Install a new custom instruction. */
13042                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
13043
13044                 /* First instruction of the group: change its type to the new custom instruction. */
13045                 instr->type = INSTR_CUSTOM_0 + i;
13046
13047                 /* All the subsequent instructions of the group: invalidate. */
13048                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
13049                         struct instruction_data *data = &p->instruction_data[j];
13050
13051                         data->invalid = 1;
13052                 }
13053
13054                 i++;
13055         }
13056
13057         /* Remove the invalidated instructions. */
13058         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
13059
13060         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
13061          * instructions that are the only instruction within their group, so they were left
13062          * unmodified).
13063          */
13064         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
13065 }
13066
13067 static int
13068 pipeline_compile(struct rte_swx_pipeline *p)
13069 {
13070         struct instruction_group_list *igl = NULL;
13071         int status = 0;
13072
13073         igl = instruction_group_list_create(p);
13074         if (!igl) {
13075                 status = -ENOMEM;
13076                 goto free;
13077         }
13078
13079         /* Code generation. */
13080         status = pipeline_codegen(p, igl);
13081         if (status)
13082                 goto free;
13083
13084         /* Build and load the shared object library. */
13085         status = pipeline_libload(p, igl);
13086         if (status)
13087                 goto free;
13088
13089         /* Adjust instructions. */
13090         status = pipeline_adjust_check(p, igl);
13091         if (status)
13092                 goto free;
13093
13094         pipeline_adjust(p, igl);
13095
13096 free:
13097         instruction_group_list_free(igl);
13098
13099         return status;
13100 }