lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8 #include <dlfcn.h>
   9
  10 #include <rte_swx_port_ethdev.h>
  11 #include <rte_swx_port_fd.h>
  12 #include <rte_swx_port_ring.h>
  13 #include "rte_swx_port_source_sink.h"
  14
  15 #include <rte_swx_table_em.h>
  16 #include <rte_swx_table_wm.h>
  17
  18 #include "rte_swx_pipeline_internal.h"
  19
  20 #define CHECK(condition, err_code)                                             \
  21 do {                                                                           \
  22         if (!(condition))                                                      \
  23                 return -(err_code);                                            \
  24 } while (0)
  25
  26 #define CHECK_NAME(name, err_code)                                             \
  27         CHECK((name) &&                                                        \
  28               (name)[0] &&                                                     \
  29               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  30               err_code)
  31
  32 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  33         CHECK((instr) &&                                                       \
  34               (instr)[0] &&                                                    \
  35               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  36                RTE_SWX_INSTRUCTION_SIZE),                                      \
  37               err_code)
  38
  39 /*
  40  * Environment.
  41  */
  42 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  43
  44 #include <rte_malloc.h>
  45
  46 static void *
  47 env_malloc(size_t size, size_t alignment, int numa_node)
  48 {
  49         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  50 }
  51
  52 static void
  53 env_free(void *start, size_t size __rte_unused)
  54 {
  55         rte_free(start);
  56 }
  57
  58 #else
  59
  60 #include <numa.h>
  61
  62 static void *
  63 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  64 {
  65         void *start;
  66
  67         if (numa_available() == -1)
  68                 return NULL;
  69
  70         start = numa_alloc_onnode(size, numa_node);
  71         if (!start)
  72                 return NULL;
  73
  74         memset(start, 0, size);
  75         return start;
  76 }
  77
  78 static void
  79 env_free(void *start, size_t size)
  80 {
  81         if (numa_available() == -1)
  82                 return;
  83
  84         numa_free(start, size);
  85 }
  86
  87 #endif
  88
  89 /*
  90  * Struct.
  91  */
  92 static struct struct_type *
  93 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  94 {
  95         struct struct_type *elem;
  96
  97         TAILQ_FOREACH(elem, &p->struct_types, node)
  98                 if (strcmp(elem->name, name) == 0)
  99                         return elem;
 100
 101         return NULL;
 102 }
 103
 104 static struct field *
 105 struct_type_field_find(struct struct_type *st, const char *name)
 106 {
 107         uint32_t i;
 108
 109         for (i = 0; i < st->n_fields; i++) {
 110                 struct field *f = &st->fields[i];
 111
 112                 if (strcmp(f->name, name) == 0)
 113                         return f;
 114         }
 115
 116         return NULL;
 117 }
 118
 119 int
 120 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 121                                       const char *name,
 122                                       struct rte_swx_field_params *fields,
 123                                       uint32_t n_fields,
 124                                       int last_field_has_variable_size)
 125 {
 126         struct struct_type *st;
 127         uint32_t i;
 128
 129         CHECK(p, EINVAL);
 130         CHECK_NAME(name, EINVAL);
 131         CHECK(fields, EINVAL);
 132         CHECK(n_fields, EINVAL);
 133
 134         for (i = 0; i < n_fields; i++) {
 135                 struct rte_swx_field_params *f = &fields[i];
 136                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 137                 uint32_t j;
 138
 139                 CHECK_NAME(f->name, EINVAL);
 140                 CHECK(f->n_bits, EINVAL);
 141                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 142                 CHECK((f->n_bits & 7) == 0, EINVAL);
 143
 144                 for (j = 0; j < i; j++) {
 145                         struct rte_swx_field_params *f_prev = &fields[j];
 146
 147                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 148                 }
 149         }
 150
 151         CHECK(!struct_type_find(p, name), EEXIST);
 152
 153         /* Node allocation. */
 154         st = calloc(1, sizeof(struct struct_type));
 155         CHECK(st, ENOMEM);
 156
 157         st->fields = calloc(n_fields, sizeof(struct field));
 158         if (!st->fields) {
 159                 free(st);
 160                 CHECK(0, ENOMEM);
 161         }
 162
 163         /* Node initialization. */
 164         strcpy(st->name, name);
 165         for (i = 0; i < n_fields; i++) {
 166                 struct field *dst = &st->fields[i];
 167                 struct rte_swx_field_params *src = &fields[i];
 168                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 169
 170                 strcpy(dst->name, src->name);
 171                 dst->n_bits = src->n_bits;
 172                 dst->offset = st->n_bits;
 173                 dst->var_size = var_size;
 174
 175                 st->n_bits += src->n_bits;
 176                 st->n_bits_min += var_size ? 0 : src->n_bits;
 177         }
 178         st->n_fields = n_fields;
 179         st->var_size = last_field_has_variable_size;
 180
 181         /* Node add to tailq. */
 182         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 183
 184         return 0;
 185 }
 186
 187 static int
 188 struct_build(struct rte_swx_pipeline *p)
 189 {
 190         uint32_t i;
 191
 192         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 193                 struct thread *t = &p->threads[i];
 194
 195                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 196                 CHECK(t->structs, ENOMEM);
 197         }
 198
 199         return 0;
 200 }
 201
 202 static void
 203 struct_build_free(struct rte_swx_pipeline *p)
 204 {
 205         uint32_t i;
 206
 207         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 208                 struct thread *t = &p->threads[i];
 209
 210                 free(t->structs);
 211                 t->structs = NULL;
 212         }
 213 }
 214
 215 static void
 216 struct_free(struct rte_swx_pipeline *p)
 217 {
 218         struct_build_free(p);
 219
 220         /* Struct types. */
 221         for ( ; ; ) {
 222                 struct struct_type *elem;
 223
 224                 elem = TAILQ_FIRST(&p->struct_types);
 225                 if (!elem)
 226                         break;
 227
 228                 TAILQ_REMOVE(&p->struct_types, elem, node);
 229                 free(elem->fields);
 230                 free(elem);
 231         }
 232 }
 233
 234 /*
 235  * Input port.
 236  */
 237 static struct port_in_type *
 238 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 239 {
 240         struct port_in_type *elem;
 241
 242         if (!name)
 243                 return NULL;
 244
 245         TAILQ_FOREACH(elem, &p->port_in_types, node)
 246                 if (strcmp(elem->name, name) == 0)
 247                         return elem;
 248
 249         return NULL;
 250 }
 251
 252 int
 253 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 254                                        const char *name,
 255                                        struct rte_swx_port_in_ops *ops)
 256 {
 257         struct port_in_type *elem;
 258
 259         CHECK(p, EINVAL);
 260         CHECK_NAME(name, EINVAL);
 261         CHECK(ops, EINVAL);
 262         CHECK(ops->create, EINVAL);
 263         CHECK(ops->free, EINVAL);
 264         CHECK(ops->pkt_rx, EINVAL);
 265         CHECK(ops->stats_read, EINVAL);
 266
 267         CHECK(!port_in_type_find(p, name), EEXIST);
 268
 269         /* Node allocation. */
 270         elem = calloc(1, sizeof(struct port_in_type));
 271         CHECK(elem, ENOMEM);
 272
 273         /* Node initialization. */
 274         strcpy(elem->name, name);
 275         memcpy(&elem->ops, ops, sizeof(*ops));
 276
 277         /* Node add to tailq. */
 278         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 279
 280         return 0;
 281 }
 282
 283 static struct port_in *
 284 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 285 {
 286         struct port_in *port;
 287
 288         TAILQ_FOREACH(port, &p->ports_in, node)
 289                 if (port->id == port_id)
 290                         return port;
 291
 292         return NULL;
 293 }
 294
 295 int
 296 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 297                                 uint32_t port_id,
 298                                 const char *port_type_name,
 299                                 void *args)
 300 {
 301         struct port_in_type *type = NULL;
 302         struct port_in *port = NULL;
 303         void *obj = NULL;
 304
 305         CHECK(p, EINVAL);
 306
 307         CHECK(!port_in_find(p, port_id), EINVAL);
 308
 309         CHECK_NAME(port_type_name, EINVAL);
 310         type = port_in_type_find(p, port_type_name);
 311         CHECK(type, EINVAL);
 312
 313         obj = type->ops.create(args);
 314         CHECK(obj, ENODEV);
 315
 316         /* Node allocation. */
 317         port = calloc(1, sizeof(struct port_in));
 318         CHECK(port, ENOMEM);
 319
 320         /* Node initialization. */
 321         port->type = type;
 322         port->obj = obj;
 323         port->id = port_id;
 324
 325         /* Node add to tailq. */
 326         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 327         if (p->n_ports_in < port_id + 1)
 328                 p->n_ports_in = port_id + 1;
 329
 330         return 0;
 331 }
 332
 333 static int
 334 port_in_build(struct rte_swx_pipeline *p)
 335 {
 336         struct port_in *port;
 337         uint32_t i;
 338
 339         CHECK(p->n_ports_in, EINVAL);
 340         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 341
 342         for (i = 0; i < p->n_ports_in; i++)
 343                 CHECK(port_in_find(p, i), EINVAL);
 344
 345         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 346         CHECK(p->in, ENOMEM);
 347
 348         TAILQ_FOREACH(port, &p->ports_in, node) {
 349                 struct port_in_runtime *in = &p->in[port->id];
 350
 351                 in->pkt_rx = port->type->ops.pkt_rx;
 352                 in->obj = port->obj;
 353         }
 354
 355         return 0;
 356 }
 357
 358 static void
 359 port_in_build_free(struct rte_swx_pipeline *p)
 360 {
 361         free(p->in);
 362         p->in = NULL;
 363 }
 364
 365 static void
 366 port_in_free(struct rte_swx_pipeline *p)
 367 {
 368         port_in_build_free(p);
 369
 370         /* Input ports. */
 371         for ( ; ; ) {
 372                 struct port_in *port;
 373
 374                 port = TAILQ_FIRST(&p->ports_in);
 375                 if (!port)
 376                         break;
 377
 378                 TAILQ_REMOVE(&p->ports_in, port, node);
 379                 port->type->ops.free(port->obj);
 380                 free(port);
 381         }
 382
 383         /* Input port types. */
 384         for ( ; ; ) {
 385                 struct port_in_type *elem;
 386
 387                 elem = TAILQ_FIRST(&p->port_in_types);
 388                 if (!elem)
 389                         break;
 390
 391                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 392                 free(elem);
 393         }
 394 }
 395
 396 /*
 397  * Output port.
 398  */
 399 static struct port_out_type *
 400 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 401 {
 402         struct port_out_type *elem;
 403
 404         if (!name)
 405                 return NULL;
 406
 407         TAILQ_FOREACH(elem, &p->port_out_types, node)
 408                 if (!strcmp(elem->name, name))
 409                         return elem;
 410
 411         return NULL;
 412 }
 413
 414 int
 415 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 416                                         const char *name,
 417                                         struct rte_swx_port_out_ops *ops)
 418 {
 419         struct port_out_type *elem;
 420
 421         CHECK(p, EINVAL);
 422         CHECK_NAME(name, EINVAL);
 423         CHECK(ops, EINVAL);
 424         CHECK(ops->create, EINVAL);
 425         CHECK(ops->free, EINVAL);
 426         CHECK(ops->pkt_tx, EINVAL);
 427         CHECK(ops->stats_read, EINVAL);
 428
 429         CHECK(!port_out_type_find(p, name), EEXIST);
 430
 431         /* Node allocation. */
 432         elem = calloc(1, sizeof(struct port_out_type));
 433         CHECK(elem, ENOMEM);
 434
 435         /* Node initialization. */
 436         strcpy(elem->name, name);
 437         memcpy(&elem->ops, ops, sizeof(*ops));
 438
 439         /* Node add to tailq. */
 440         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 441
 442         return 0;
 443 }
 444
 445 static struct port_out *
 446 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 447 {
 448         struct port_out *port;
 449
 450         TAILQ_FOREACH(port, &p->ports_out, node)
 451                 if (port->id == port_id)
 452                         return port;
 453
 454         return NULL;
 455 }
 456
 457 int
 458 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 459                                  uint32_t port_id,
 460                                  const char *port_type_name,
 461                                  void *args)
 462 {
 463         struct port_out_type *type = NULL;
 464         struct port_out *port = NULL;
 465         void *obj = NULL;
 466
 467         CHECK(p, EINVAL);
 468
 469         CHECK(!port_out_find(p, port_id), EINVAL);
 470
 471         CHECK_NAME(port_type_name, EINVAL);
 472         type = port_out_type_find(p, port_type_name);
 473         CHECK(type, EINVAL);
 474
 475         obj = type->ops.create(args);
 476         CHECK(obj, ENODEV);
 477
 478         /* Node allocation. */
 479         port = calloc(1, sizeof(struct port_out));
 480         CHECK(port, ENOMEM);
 481
 482         /* Node initialization. */
 483         port->type = type;
 484         port->obj = obj;
 485         port->id = port_id;
 486
 487         /* Node add to tailq. */
 488         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 489         if (p->n_ports_out < port_id + 1)
 490                 p->n_ports_out = port_id + 1;
 491
 492         return 0;
 493 }
 494
 495 static int
 496 port_out_build(struct rte_swx_pipeline *p)
 497 {
 498         struct port_out *port;
 499         uint32_t i;
 500
 501         CHECK(p->n_ports_out, EINVAL);
 502
 503         for (i = 0; i < p->n_ports_out; i++)
 504                 CHECK(port_out_find(p, i), EINVAL);
 505
 506         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 507         CHECK(p->out, ENOMEM);
 508
 509         TAILQ_FOREACH(port, &p->ports_out, node) {
 510                 struct port_out_runtime *out = &p->out[port->id];
 511
 512                 out->pkt_tx = port->type->ops.pkt_tx;
 513                 out->flush = port->type->ops.flush;
 514                 out->obj = port->obj;
 515         }
 516
 517         return 0;
 518 }
 519
 520 static void
 521 port_out_build_free(struct rte_swx_pipeline *p)
 522 {
 523         free(p->out);
 524         p->out = NULL;
 525 }
 526
 527 static void
 528 port_out_free(struct rte_swx_pipeline *p)
 529 {
 530         port_out_build_free(p);
 531
 532         /* Output ports. */
 533         for ( ; ; ) {
 534                 struct port_out *port;
 535
 536                 port = TAILQ_FIRST(&p->ports_out);
 537                 if (!port)
 538                         break;
 539
 540                 TAILQ_REMOVE(&p->ports_out, port, node);
 541                 port->type->ops.free(port->obj);
 542                 free(port);
 543         }
 544
 545         /* Output port types. */
 546         for ( ; ; ) {
 547                 struct port_out_type *elem;
 548
 549                 elem = TAILQ_FIRST(&p->port_out_types);
 550                 if (!elem)
 551                         break;
 552
 553                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 554                 free(elem);
 555         }
 556 }
 557
 558 /*
 559  * Extern object.
 560  */
 561 static struct extern_type *
 562 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 563 {
 564         struct extern_type *elem;
 565
 566         TAILQ_FOREACH(elem, &p->extern_types, node)
 567                 if (strcmp(elem->name, name) == 0)
 568                         return elem;
 569
 570         return NULL;
 571 }
 572
 573 static struct extern_type_member_func *
 574 extern_type_member_func_find(struct extern_type *type, const char *name)
 575 {
 576         struct extern_type_member_func *elem;
 577
 578         TAILQ_FOREACH(elem, &type->funcs, node)
 579                 if (strcmp(elem->name, name) == 0)
 580                         return elem;
 581
 582         return NULL;
 583 }
 584
 585 static struct extern_obj *
 586 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 587 {
 588         struct extern_obj *elem;
 589
 590         TAILQ_FOREACH(elem, &p->extern_objs, node)
 591                 if (strcmp(elem->name, name) == 0)
 592                         return elem;
 593
 594         return NULL;
 595 }
 596
 597 static struct extern_type_member_func *
 598 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 599                              const char *name,
 600                              struct extern_obj **obj)
 601 {
 602         struct extern_obj *object;
 603         struct extern_type_member_func *func;
 604         char *object_name, *func_name;
 605
 606         if (name[0] != 'e' || name[1] != '.')
 607                 return NULL;
 608
 609         object_name = strdup(&name[2]);
 610         if (!object_name)
 611                 return NULL;
 612
 613         func_name = strchr(object_name, '.');
 614         if (!func_name) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         *func_name = 0;
 620         func_name++;
 621
 622         object = extern_obj_find(p, object_name);
 623         if (!object) {
 624                 free(object_name);
 625                 return NULL;
 626         }
 627
 628         func = extern_type_member_func_find(object->type, func_name);
 629         if (!func) {
 630                 free(object_name);
 631                 return NULL;
 632         }
 633
 634         if (obj)
 635                 *obj = object;
 636
 637         free(object_name);
 638         return func;
 639 }
 640
 641 static struct field *
 642 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 643                                const char *name,
 644                                struct extern_obj **object)
 645 {
 646         struct extern_obj *obj;
 647         struct field *f;
 648         char *obj_name, *field_name;
 649
 650         if ((name[0] != 'e') || (name[1] != '.'))
 651                 return NULL;
 652
 653         obj_name = strdup(&name[2]);
 654         if (!obj_name)
 655                 return NULL;
 656
 657         field_name = strchr(obj_name, '.');
 658         if (!field_name) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         *field_name = 0;
 664         field_name++;
 665
 666         obj = extern_obj_find(p, obj_name);
 667         if (!obj) {
 668                 free(obj_name);
 669                 return NULL;
 670         }
 671
 672         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 673         if (!f) {
 674                 free(obj_name);
 675                 return NULL;
 676         }
 677
 678         if (object)
 679                 *object = obj;
 680
 681         free(obj_name);
 682         return f;
 683 }
 684
 685 int
 686 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 687         const char *name,
 688         const char *mailbox_struct_type_name,
 689         rte_swx_extern_type_constructor_t constructor,
 690         rte_swx_extern_type_destructor_t destructor)
 691 {
 692         struct extern_type *elem;
 693         struct struct_type *mailbox_struct_type;
 694
 695         CHECK(p, EINVAL);
 696
 697         CHECK_NAME(name, EINVAL);
 698         CHECK(!extern_type_find(p, name), EEXIST);
 699
 700         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 701         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 702         CHECK(mailbox_struct_type, EINVAL);
 703         CHECK(!mailbox_struct_type->var_size, EINVAL);
 704
 705         CHECK(constructor, EINVAL);
 706         CHECK(destructor, EINVAL);
 707
 708         /* Node allocation. */
 709         elem = calloc(1, sizeof(struct extern_type));
 710         CHECK(elem, ENOMEM);
 711
 712         /* Node initialization. */
 713         strcpy(elem->name, name);
 714         elem->mailbox_struct_type = mailbox_struct_type;
 715         elem->constructor = constructor;
 716         elem->destructor = destructor;
 717         TAILQ_INIT(&elem->funcs);
 718
 719         /* Node add to tailq. */
 720         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 721
 722         return 0;
 723 }
 724
 725 int
 726 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 727         const char *extern_type_name,
 728         const char *name,
 729         rte_swx_extern_type_member_func_t member_func)
 730 {
 731         struct extern_type *type;
 732         struct extern_type_member_func *type_member;
 733
 734         CHECK(p, EINVAL);
 735
 736         CHECK_NAME(extern_type_name, EINVAL);
 737         type = extern_type_find(p, extern_type_name);
 738         CHECK(type, EINVAL);
 739         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 740
 741         CHECK_NAME(name, EINVAL);
 742         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 743
 744         CHECK(member_func, EINVAL);
 745
 746         /* Node allocation. */
 747         type_member = calloc(1, sizeof(struct extern_type_member_func));
 748         CHECK(type_member, ENOMEM);
 749
 750         /* Node initialization. */
 751         strcpy(type_member->name, name);
 752         type_member->func = member_func;
 753         type_member->id = type->n_funcs;
 754
 755         /* Node add to tailq. */
 756         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 757         type->n_funcs++;
 758
 759         return 0;
 760 }
 761
 762 int
 763 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 764                                       const char *extern_type_name,
 765                                       const char *name,
 766                                       const char *args)
 767 {
 768         struct extern_type *type;
 769         struct extern_obj *obj;
 770         void *obj_handle;
 771
 772         CHECK(p, EINVAL);
 773
 774         CHECK_NAME(extern_type_name, EINVAL);
 775         type = extern_type_find(p, extern_type_name);
 776         CHECK(type, EINVAL);
 777
 778         CHECK_NAME(name, EINVAL);
 779         CHECK(!extern_obj_find(p, name), EEXIST);
 780
 781         /* Node allocation. */
 782         obj = calloc(1, sizeof(struct extern_obj));
 783         CHECK(obj, ENOMEM);
 784
 785         /* Object construction. */
 786         obj_handle = type->constructor(args);
 787         if (!obj_handle) {
 788                 free(obj);
 789                 CHECK(0, ENODEV);
 790         }
 791
 792         /* Node initialization. */
 793         strcpy(obj->name, name);
 794         obj->type = type;
 795         obj->obj = obj_handle;
 796         obj->struct_id = p->n_structs;
 797         obj->id = p->n_extern_objs;
 798
 799         /* Node add to tailq. */
 800         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 801         p->n_extern_objs++;
 802         p->n_structs++;
 803
 804         return 0;
 805 }
 806
 807 static int
 808 extern_obj_build(struct rte_swx_pipeline *p)
 809 {
 810         uint32_t i;
 811
 812         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 813                 struct thread *t = &p->threads[i];
 814                 struct extern_obj *obj;
 815
 816                 t->extern_objs = calloc(p->n_extern_objs,
 817                                         sizeof(struct extern_obj_runtime));
 818                 CHECK(t->extern_objs, ENOMEM);
 819
 820                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 821                         struct extern_obj_runtime *r =
 822                                 &t->extern_objs[obj->id];
 823                         struct extern_type_member_func *func;
 824                         uint32_t mailbox_size =
 825                                 obj->type->mailbox_struct_type->n_bits / 8;
 826
 827                         r->obj = obj->obj;
 828
 829                         r->mailbox = calloc(1, mailbox_size);
 830                         CHECK(r->mailbox, ENOMEM);
 831
 832                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 833                                 r->funcs[func->id] = func->func;
 834
 835                         t->structs[obj->struct_id] = r->mailbox;
 836                 }
 837         }
 838
 839         return 0;
 840 }
 841
 842 static void
 843 extern_obj_build_free(struct rte_swx_pipeline *p)
 844 {
 845         uint32_t i;
 846
 847         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 848                 struct thread *t = &p->threads[i];
 849                 uint32_t j;
 850
 851                 if (!t->extern_objs)
 852                         continue;
 853
 854                 for (j = 0; j < p->n_extern_objs; j++) {
 855                         struct extern_obj_runtime *r = &t->extern_objs[j];
 856
 857                         free(r->mailbox);
 858                 }
 859
 860                 free(t->extern_objs);
 861                 t->extern_objs = NULL;
 862         }
 863 }
 864
 865 static void
 866 extern_obj_free(struct rte_swx_pipeline *p)
 867 {
 868         extern_obj_build_free(p);
 869
 870         /* Extern objects. */
 871         for ( ; ; ) {
 872                 struct extern_obj *elem;
 873
 874                 elem = TAILQ_FIRST(&p->extern_objs);
 875                 if (!elem)
 876                         break;
 877
 878                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 879                 if (elem->obj)
 880                         elem->type->destructor(elem->obj);
 881                 free(elem);
 882         }
 883
 884         /* Extern types. */
 885         for ( ; ; ) {
 886                 struct extern_type *elem;
 887
 888                 elem = TAILQ_FIRST(&p->extern_types);
 889                 if (!elem)
 890                         break;
 891
 892                 TAILQ_REMOVE(&p->extern_types, elem, node);
 893
 894                 for ( ; ; ) {
 895                         struct extern_type_member_func *func;
 896
 897                         func = TAILQ_FIRST(&elem->funcs);
 898                         if (!func)
 899                                 break;
 900
 901                         TAILQ_REMOVE(&elem->funcs, func, node);
 902                         free(func);
 903                 }
 904
 905                 free(elem);
 906         }
 907 }
 908
 909 /*
 910  * Extern function.
 911  */
 912 static struct extern_func *
 913 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 914 {
 915         struct extern_func *elem;
 916
 917         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 918                 if (strcmp(elem->name, name) == 0)
 919                         return elem;
 920
 921         return NULL;
 922 }
 923
 924 static struct extern_func *
 925 extern_func_parse(struct rte_swx_pipeline *p,
 926                   const char *name)
 927 {
 928         if (name[0] != 'f' || name[1] != '.')
 929                 return NULL;
 930
 931         return extern_func_find(p, &name[2]);
 932 }
 933
 934 static struct field *
 935 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 936                                 const char *name,
 937                                 struct extern_func **function)
 938 {
 939         struct extern_func *func;
 940         struct field *f;
 941         char *func_name, *field_name;
 942
 943         if ((name[0] != 'f') || (name[1] != '.'))
 944                 return NULL;
 945
 946         func_name = strdup(&name[2]);
 947         if (!func_name)
 948                 return NULL;
 949
 950         field_name = strchr(func_name, '.');
 951         if (!field_name) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         *field_name = 0;
 957         field_name++;
 958
 959         func = extern_func_find(p, func_name);
 960         if (!func) {
 961                 free(func_name);
 962                 return NULL;
 963         }
 964
 965         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 966         if (!f) {
 967                 free(func_name);
 968                 return NULL;
 969         }
 970
 971         if (function)
 972                 *function = func;
 973
 974         free(func_name);
 975         return f;
 976 }
 977
 978 int
 979 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 980                                       const char *name,
 981                                       const char *mailbox_struct_type_name,
 982                                       rte_swx_extern_func_t func)
 983 {
 984         struct extern_func *f;
 985         struct struct_type *mailbox_struct_type;
 986
 987         CHECK(p, EINVAL);
 988
 989         CHECK_NAME(name, EINVAL);
 990         CHECK(!extern_func_find(p, name), EEXIST);
 991
 992         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 993         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 994         CHECK(mailbox_struct_type, EINVAL);
 995         CHECK(!mailbox_struct_type->var_size, EINVAL);
 996
 997         CHECK(func, EINVAL);
 998
 999         /* Node allocation. */
1000         f = calloc(1, sizeof(struct extern_func));
1001         CHECK(func, ENOMEM);
1002
1003         /* Node initialization. */
1004         strcpy(f->name, name);
1005         f->mailbox_struct_type = mailbox_struct_type;
1006         f->func = func;
1007         f->struct_id = p->n_structs;
1008         f->id = p->n_extern_funcs;
1009
1010         /* Node add to tailq. */
1011         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1012         p->n_extern_funcs++;
1013         p->n_structs++;
1014
1015         return 0;
1016 }
1017
1018 static int
1019 extern_func_build(struct rte_swx_pipeline *p)
1020 {
1021         uint32_t i;
1022
1023         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1024                 struct thread *t = &p->threads[i];
1025                 struct extern_func *func;
1026
1027                 /* Memory allocation. */
1028                 t->extern_funcs = calloc(p->n_extern_funcs,
1029                                          sizeof(struct extern_func_runtime));
1030                 CHECK(t->extern_funcs, ENOMEM);
1031
1032                 /* Extern function. */
1033                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1034                         struct extern_func_runtime *r =
1035                                 &t->extern_funcs[func->id];
1036                         uint32_t mailbox_size =
1037                                 func->mailbox_struct_type->n_bits / 8;
1038
1039                         r->func = func->func;
1040
1041                         r->mailbox = calloc(1, mailbox_size);
1042                         CHECK(r->mailbox, ENOMEM);
1043
1044                         t->structs[func->struct_id] = r->mailbox;
1045                 }
1046         }
1047
1048         return 0;
1049 }
1050
1051 static void
1052 extern_func_build_free(struct rte_swx_pipeline *p)
1053 {
1054         uint32_t i;
1055
1056         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1057                 struct thread *t = &p->threads[i];
1058                 uint32_t j;
1059
1060                 if (!t->extern_funcs)
1061                         continue;
1062
1063                 for (j = 0; j < p->n_extern_funcs; j++) {
1064                         struct extern_func_runtime *r = &t->extern_funcs[j];
1065
1066                         free(r->mailbox);
1067                 }
1068
1069                 free(t->extern_funcs);
1070                 t->extern_funcs = NULL;
1071         }
1072 }
1073
1074 static void
1075 extern_func_free(struct rte_swx_pipeline *p)
1076 {
1077         extern_func_build_free(p);
1078
1079         for ( ; ; ) {
1080                 struct extern_func *elem;
1081
1082                 elem = TAILQ_FIRST(&p->extern_funcs);
1083                 if (!elem)
1084                         break;
1085
1086                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1087                 free(elem);
1088         }
1089 }
1090
1091 /*
1092  * Header.
1093  */
1094 static struct header *
1095 header_find(struct rte_swx_pipeline *p, const char *name)
1096 {
1097         struct header *elem;
1098
1099         TAILQ_FOREACH(elem, &p->headers, node)
1100                 if (strcmp(elem->name, name) == 0)
1101                         return elem;
1102
1103         return NULL;
1104 }
1105
1106 static struct header *
1107 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1108 {
1109         struct header *elem;
1110
1111         TAILQ_FOREACH(elem, &p->headers, node)
1112                 if (elem->struct_id == struct_id)
1113                         return elem;
1114
1115         return NULL;
1116 }
1117
1118 static struct header *
1119 header_parse(struct rte_swx_pipeline *p,
1120              const char *name)
1121 {
1122         if (name[0] != 'h' || name[1] != '.')
1123                 return NULL;
1124
1125         return header_find(p, &name[2]);
1126 }
1127
1128 static struct field *
1129 header_field_parse(struct rte_swx_pipeline *p,
1130                    const char *name,
1131                    struct header **header)
1132 {
1133         struct header *h;
1134         struct field *f;
1135         char *header_name, *field_name;
1136
1137         if ((name[0] != 'h') || (name[1] != '.'))
1138                 return NULL;
1139
1140         header_name = strdup(&name[2]);
1141         if (!header_name)
1142                 return NULL;
1143
1144         field_name = strchr(header_name, '.');
1145         if (!field_name) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         *field_name = 0;
1151         field_name++;
1152
1153         h = header_find(p, header_name);
1154         if (!h) {
1155                 free(header_name);
1156                 return NULL;
1157         }
1158
1159         f = struct_type_field_find(h->st, field_name);
1160         if (!f) {
1161                 free(header_name);
1162                 return NULL;
1163         }
1164
1165         if (header)
1166                 *header = h;
1167
1168         free(header_name);
1169         return f;
1170 }
1171
1172 int
1173 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1174                                         const char *name,
1175                                         const char *struct_type_name)
1176 {
1177         struct struct_type *st;
1178         struct header *h;
1179         size_t n_headers_max;
1180
1181         CHECK(p, EINVAL);
1182         CHECK_NAME(name, EINVAL);
1183         CHECK_NAME(struct_type_name, EINVAL);
1184
1185         CHECK(!header_find(p, name), EEXIST);
1186
1187         st = struct_type_find(p, struct_type_name);
1188         CHECK(st, EINVAL);
1189
1190         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1191         CHECK(p->n_headers < n_headers_max, ENOSPC);
1192
1193         /* Node allocation. */
1194         h = calloc(1, sizeof(struct header));
1195         CHECK(h, ENOMEM);
1196
1197         /* Node initialization. */
1198         strcpy(h->name, name);
1199         h->st = st;
1200         h->struct_id = p->n_structs;
1201         h->id = p->n_headers;
1202
1203         /* Node add to tailq. */
1204         TAILQ_INSERT_TAIL(&p->headers, h, node);
1205         p->n_headers++;
1206         p->n_structs++;
1207
1208         return 0;
1209 }
1210
1211 static int
1212 header_build(struct rte_swx_pipeline *p)
1213 {
1214         struct header *h;
1215         uint32_t n_bytes = 0, i;
1216
1217         TAILQ_FOREACH(h, &p->headers, node) {
1218                 n_bytes += h->st->n_bits / 8;
1219         }
1220
1221         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1222                 struct thread *t = &p->threads[i];
1223                 uint32_t offset = 0;
1224
1225                 t->headers = calloc(p->n_headers,
1226                                     sizeof(struct header_runtime));
1227                 CHECK(t->headers, ENOMEM);
1228
1229                 t->headers_out = calloc(p->n_headers,
1230                                         sizeof(struct header_out_runtime));
1231                 CHECK(t->headers_out, ENOMEM);
1232
1233                 t->header_storage = calloc(1, n_bytes);
1234                 CHECK(t->header_storage, ENOMEM);
1235
1236                 t->header_out_storage = calloc(1, n_bytes);
1237                 CHECK(t->header_out_storage, ENOMEM);
1238
1239                 TAILQ_FOREACH(h, &p->headers, node) {
1240                         uint8_t *header_storage;
1241                         uint32_t n_bytes =  h->st->n_bits / 8;
1242
1243                         header_storage = &t->header_storage[offset];
1244                         offset += n_bytes;
1245
1246                         t->headers[h->id].ptr0 = header_storage;
1247                         t->headers[h->id].n_bytes = n_bytes;
1248
1249                         t->structs[h->struct_id] = header_storage;
1250                 }
1251         }
1252
1253         return 0;
1254 }
1255
1256 static void
1257 header_build_free(struct rte_swx_pipeline *p)
1258 {
1259         uint32_t i;
1260
1261         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1262                 struct thread *t = &p->threads[i];
1263
1264                 free(t->headers_out);
1265                 t->headers_out = NULL;
1266
1267                 free(t->headers);
1268                 t->headers = NULL;
1269
1270                 free(t->header_out_storage);
1271                 t->header_out_storage = NULL;
1272
1273                 free(t->header_storage);
1274                 t->header_storage = NULL;
1275         }
1276 }
1277
1278 static void
1279 header_free(struct rte_swx_pipeline *p)
1280 {
1281         header_build_free(p);
1282
1283         for ( ; ; ) {
1284                 struct header *elem;
1285
1286                 elem = TAILQ_FIRST(&p->headers);
1287                 if (!elem)
1288                         break;
1289
1290                 TAILQ_REMOVE(&p->headers, elem, node);
1291                 free(elem);
1292         }
1293 }
1294
1295 /*
1296  * Meta-data.
1297  */
1298 static struct field *
1299 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1300 {
1301         if (!p->metadata_st)
1302                 return NULL;
1303
1304         if (name[0] != 'm' || name[1] != '.')
1305                 return NULL;
1306
1307         return struct_type_field_find(p->metadata_st, &name[2]);
1308 }
1309
1310 int
1311 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1312                                           const char *struct_type_name)
1313 {
1314         struct struct_type *st = NULL;
1315
1316         CHECK(p, EINVAL);
1317
1318         CHECK_NAME(struct_type_name, EINVAL);
1319         st  = struct_type_find(p, struct_type_name);
1320         CHECK(st, EINVAL);
1321         CHECK(!st->var_size, EINVAL);
1322         CHECK(!p->metadata_st, EINVAL);
1323
1324         p->metadata_st = st;
1325         p->metadata_struct_id = p->n_structs;
1326
1327         p->n_structs++;
1328
1329         return 0;
1330 }
1331
1332 static int
1333 metadata_build(struct rte_swx_pipeline *p)
1334 {
1335         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1336         uint32_t i;
1337
1338         /* Thread-level initialization. */
1339         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1340                 struct thread *t = &p->threads[i];
1341                 uint8_t *metadata;
1342
1343                 metadata = calloc(1, n_bytes);
1344                 CHECK(metadata, ENOMEM);
1345
1346                 t->metadata = metadata;
1347                 t->structs[p->metadata_struct_id] = metadata;
1348         }
1349
1350         return 0;
1351 }
1352
1353 static void
1354 metadata_build_free(struct rte_swx_pipeline *p)
1355 {
1356         uint32_t i;
1357
1358         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1359                 struct thread *t = &p->threads[i];
1360
1361                 free(t->metadata);
1362                 t->metadata = NULL;
1363         }
1364 }
1365
1366 static void
1367 metadata_free(struct rte_swx_pipeline *p)
1368 {
1369         metadata_build_free(p);
1370 }
1371
1372 /*
1373  * Instruction.
1374  */
1375 static int
1376 instruction_is_tx(enum instruction_type type)
1377 {
1378         switch (type) {
1379         case INSTR_TX:
1380         case INSTR_TX_I:
1381         case INSTR_DROP:
1382                 return 1;
1383
1384         default:
1385                 return 0;
1386         }
1387 }
1388
1389 static int
1390 instruction_does_tx(struct instruction *instr)
1391 {
1392         switch (instr->type) {
1393         case INSTR_TX:
1394         case INSTR_TX_I:
1395         case INSTR_DROP:
1396         case INSTR_HDR_EMIT_TX:
1397         case INSTR_HDR_EMIT2_TX:
1398         case INSTR_HDR_EMIT3_TX:
1399         case INSTR_HDR_EMIT4_TX:
1400         case INSTR_HDR_EMIT5_TX:
1401         case INSTR_HDR_EMIT6_TX:
1402         case INSTR_HDR_EMIT7_TX:
1403         case INSTR_HDR_EMIT8_TX:
1404                 return 1;
1405         default:
1406                 return 0;
1407         }
1408 }
1409
1410 static int
1411 instruction_is_jmp(struct instruction *instr)
1412 {
1413         switch (instr->type) {
1414         case INSTR_JMP:
1415         case INSTR_JMP_VALID:
1416         case INSTR_JMP_INVALID:
1417         case INSTR_JMP_HIT:
1418         case INSTR_JMP_MISS:
1419         case INSTR_JMP_ACTION_HIT:
1420         case INSTR_JMP_ACTION_MISS:
1421         case INSTR_JMP_EQ:
1422         case INSTR_JMP_EQ_MH:
1423         case INSTR_JMP_EQ_HM:
1424         case INSTR_JMP_EQ_HH:
1425         case INSTR_JMP_EQ_I:
1426         case INSTR_JMP_NEQ:
1427         case INSTR_JMP_NEQ_MH:
1428         case INSTR_JMP_NEQ_HM:
1429         case INSTR_JMP_NEQ_HH:
1430         case INSTR_JMP_NEQ_I:
1431         case INSTR_JMP_LT:
1432         case INSTR_JMP_LT_MH:
1433         case INSTR_JMP_LT_HM:
1434         case INSTR_JMP_LT_HH:
1435         case INSTR_JMP_LT_MI:
1436         case INSTR_JMP_LT_HI:
1437         case INSTR_JMP_GT:
1438         case INSTR_JMP_GT_MH:
1439         case INSTR_JMP_GT_HM:
1440         case INSTR_JMP_GT_HH:
1441         case INSTR_JMP_GT_MI:
1442         case INSTR_JMP_GT_HI:
1443                 return 1;
1444
1445         default:
1446                 return 0;
1447         }
1448 }
1449
1450 static int
1451 instruction_does_thread_yield(struct instruction *instr)
1452 {
1453         switch (instr->type) {
1454         case INSTR_RX:
1455         case INSTR_TABLE:
1456         case INSTR_TABLE_AF:
1457         case INSTR_SELECTOR:
1458         case INSTR_LEARNER:
1459         case INSTR_LEARNER_AF:
1460         case INSTR_EXTERN_OBJ:
1461         case INSTR_EXTERN_FUNC:
1462                 return 1;
1463         default:
1464                 return 0;
1465         }
1466 }
1467
1468 static struct field *
1469 action_field_parse(struct action *action, const char *name);
1470
1471 static struct field *
1472 struct_field_parse(struct rte_swx_pipeline *p,
1473                    struct action *action,
1474                    const char *name,
1475                    uint32_t *struct_id)
1476 {
1477         struct field *f;
1478
1479         switch (name[0]) {
1480         case 'h':
1481         {
1482                 struct header *header;
1483
1484                 f = header_field_parse(p, name, &header);
1485                 if (!f)
1486                         return NULL;
1487
1488                 *struct_id = header->struct_id;
1489                 return f;
1490         }
1491
1492         case 'm':
1493         {
1494                 f = metadata_field_parse(p, name);
1495                 if (!f)
1496                         return NULL;
1497
1498                 *struct_id = p->metadata_struct_id;
1499                 return f;
1500         }
1501
1502         case 't':
1503         {
1504                 if (!action)
1505                         return NULL;
1506
1507                 f = action_field_parse(action, name);
1508                 if (!f)
1509                         return NULL;
1510
1511                 *struct_id = 0;
1512                 return f;
1513         }
1514
1515         case 'e':
1516         {
1517                 struct extern_obj *obj;
1518
1519                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1520                 if (!f)
1521                         return NULL;
1522
1523                 *struct_id = obj->struct_id;
1524                 return f;
1525         }
1526
1527         case 'f':
1528         {
1529                 struct extern_func *func;
1530
1531                 f = extern_func_mailbox_field_parse(p, name, &func);
1532                 if (!f)
1533                         return NULL;
1534
1535                 *struct_id = func->struct_id;
1536                 return f;
1537         }
1538
1539         default:
1540                 return NULL;
1541         }
1542 }
1543
1544 /*
1545  * rx.
1546  */
1547 static int
1548 instr_rx_translate(struct rte_swx_pipeline *p,
1549                    struct action *action,
1550                    char **tokens,
1551                    int n_tokens,
1552                    struct instruction *instr,
1553                    struct instruction_data *data __rte_unused)
1554 {
1555         struct field *f;
1556
1557         CHECK(!action, EINVAL);
1558         CHECK(n_tokens == 2, EINVAL);
1559
1560         f = metadata_field_parse(p, tokens[1]);
1561         CHECK(f, EINVAL);
1562
1563         instr->type = INSTR_RX;
1564         instr->io.io.offset = f->offset / 8;
1565         instr->io.io.n_bits = f->n_bits;
1566         return 0;
1567 }
1568
1569 /*
1570  * tx.
1571  */
1572 static int
1573 instr_tx_translate(struct rte_swx_pipeline *p,
1574                    struct action *action __rte_unused,
1575                    char **tokens,
1576                    int n_tokens,
1577                    struct instruction *instr,
1578                    struct instruction_data *data __rte_unused)
1579 {
1580         char *port = tokens[1];
1581         struct field *f;
1582         uint32_t port_val;
1583
1584         CHECK(n_tokens == 2, EINVAL);
1585
1586         f = metadata_field_parse(p, port);
1587         if (f) {
1588                 instr->type = INSTR_TX;
1589                 instr->io.io.offset = f->offset / 8;
1590                 instr->io.io.n_bits = f->n_bits;
1591                 return 0;
1592         }
1593
1594         /* TX_I. */
1595         port_val = strtoul(port, &port, 0);
1596         CHECK(!port[0], EINVAL);
1597
1598         instr->type = INSTR_TX_I;
1599         instr->io.io.val = port_val;
1600         return 0;
1601 }
1602
1603 static int
1604 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1605                      struct action *action __rte_unused,
1606                      char **tokens __rte_unused,
1607                      int n_tokens,
1608                      struct instruction *instr,
1609                      struct instruction_data *data __rte_unused)
1610 {
1611         CHECK(n_tokens == 1, EINVAL);
1612
1613         /* DROP. */
1614         instr->type = INSTR_DROP;
1615         return 0;
1616 }
1617
1618 static inline void
1619 instr_tx_exec(struct rte_swx_pipeline *p)
1620 {
1621         struct thread *t = &p->threads[p->thread_id];
1622         struct instruction *ip = t->ip;
1623
1624         __instr_tx_exec(p, t, ip);
1625
1626         /* Thread. */
1627         thread_ip_reset(p, t);
1628         instr_rx_exec(p);
1629 }
1630
1631 static inline void
1632 instr_tx_i_exec(struct rte_swx_pipeline *p)
1633 {
1634         struct thread *t = &p->threads[p->thread_id];
1635         struct instruction *ip = t->ip;
1636
1637         __instr_tx_i_exec(p, t, ip);
1638
1639         /* Thread. */
1640         thread_ip_reset(p, t);
1641         instr_rx_exec(p);
1642 }
1643
1644 static inline void
1645 instr_drop_exec(struct rte_swx_pipeline *p)
1646 {
1647         struct thread *t = &p->threads[p->thread_id];
1648         struct instruction *ip = t->ip;
1649
1650         __instr_drop_exec(p, t, ip);
1651
1652         /* Thread. */
1653         thread_ip_reset(p, t);
1654         instr_rx_exec(p);
1655 }
1656
1657 /*
1658  * extract.
1659  */
1660 static int
1661 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1662                             struct action *action,
1663                             char **tokens,
1664                             int n_tokens,
1665                             struct instruction *instr,
1666                             struct instruction_data *data __rte_unused)
1667 {
1668         struct header *h;
1669
1670         CHECK(!action, EINVAL);
1671         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1672
1673         h = header_parse(p, tokens[1]);
1674         CHECK(h, EINVAL);
1675
1676         if (n_tokens == 2) {
1677                 CHECK(!h->st->var_size, EINVAL);
1678
1679                 instr->type = INSTR_HDR_EXTRACT;
1680                 instr->io.hdr.header_id[0] = h->id;
1681                 instr->io.hdr.struct_id[0] = h->struct_id;
1682                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1683         } else {
1684                 struct field *mf;
1685
1686                 CHECK(h->st->var_size, EINVAL);
1687
1688                 mf = metadata_field_parse(p, tokens[2]);
1689                 CHECK(mf, EINVAL);
1690                 CHECK(!mf->var_size, EINVAL);
1691
1692                 instr->type = INSTR_HDR_EXTRACT_M;
1693                 instr->io.io.offset = mf->offset / 8;
1694                 instr->io.io.n_bits = mf->n_bits;
1695                 instr->io.hdr.header_id[0] = h->id;
1696                 instr->io.hdr.struct_id[0] = h->struct_id;
1697                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1698         }
1699
1700         return 0;
1701 }
1702
1703 static int
1704 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1705                               struct action *action,
1706                               char **tokens,
1707                               int n_tokens,
1708                               struct instruction *instr,
1709                               struct instruction_data *data __rte_unused)
1710 {
1711         struct header *h;
1712
1713         CHECK(!action, EINVAL);
1714         CHECK(n_tokens == 2, EINVAL);
1715
1716         h = header_parse(p, tokens[1]);
1717         CHECK(h, EINVAL);
1718         CHECK(!h->st->var_size, EINVAL);
1719
1720         instr->type = INSTR_HDR_LOOKAHEAD;
1721         instr->io.hdr.header_id[0] = h->id;
1722         instr->io.hdr.struct_id[0] = h->struct_id;
1723         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1724
1725         return 0;
1726 }
1727
1728 static inline void
1729 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1730 {
1731         struct thread *t = &p->threads[p->thread_id];
1732         struct instruction *ip = t->ip;
1733
1734         __instr_hdr_extract_exec(p, t, ip);
1735
1736         /* Thread. */
1737         thread_ip_inc(p);
1738 }
1739
1740 static inline void
1741 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1742 {
1743         struct thread *t = &p->threads[p->thread_id];
1744         struct instruction *ip = t->ip;
1745
1746         __instr_hdr_extract2_exec(p, t, ip);
1747
1748         /* Thread. */
1749         thread_ip_inc(p);
1750 }
1751
1752 static inline void
1753 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1754 {
1755         struct thread *t = &p->threads[p->thread_id];
1756         struct instruction *ip = t->ip;
1757
1758         __instr_hdr_extract3_exec(p, t, ip);
1759
1760         /* Thread. */
1761         thread_ip_inc(p);
1762 }
1763
1764 static inline void
1765 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1766 {
1767         struct thread *t = &p->threads[p->thread_id];
1768         struct instruction *ip = t->ip;
1769
1770         __instr_hdr_extract4_exec(p, t, ip);
1771
1772         /* Thread. */
1773         thread_ip_inc(p);
1774 }
1775
1776 static inline void
1777 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1778 {
1779         struct thread *t = &p->threads[p->thread_id];
1780         struct instruction *ip = t->ip;
1781
1782         __instr_hdr_extract5_exec(p, t, ip);
1783
1784         /* Thread. */
1785         thread_ip_inc(p);
1786 }
1787
1788 static inline void
1789 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1790 {
1791         struct thread *t = &p->threads[p->thread_id];
1792         struct instruction *ip = t->ip;
1793
1794         __instr_hdr_extract6_exec(p, t, ip);
1795
1796         /* Thread. */
1797         thread_ip_inc(p);
1798 }
1799
1800 static inline void
1801 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1802 {
1803         struct thread *t = &p->threads[p->thread_id];
1804         struct instruction *ip = t->ip;
1805
1806         __instr_hdr_extract7_exec(p, t, ip);
1807
1808         /* Thread. */
1809         thread_ip_inc(p);
1810 }
1811
1812 static inline void
1813 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1814 {
1815         struct thread *t = &p->threads[p->thread_id];
1816         struct instruction *ip = t->ip;
1817
1818         __instr_hdr_extract8_exec(p, t, ip);
1819
1820         /* Thread. */
1821         thread_ip_inc(p);
1822 }
1823
1824 static inline void
1825 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828         struct instruction *ip = t->ip;
1829
1830         __instr_hdr_extract_m_exec(p, t, ip);
1831
1832         /* Thread. */
1833         thread_ip_inc(p);
1834 }
1835
1836 static inline void
1837 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1838 {
1839         struct thread *t = &p->threads[p->thread_id];
1840         struct instruction *ip = t->ip;
1841
1842         __instr_hdr_lookahead_exec(p, t, ip);
1843
1844         /* Thread. */
1845         thread_ip_inc(p);
1846 }
1847
1848 /*
1849  * emit.
1850  */
1851 static int
1852 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1853                          struct action *action __rte_unused,
1854                          char **tokens,
1855                          int n_tokens,
1856                          struct instruction *instr,
1857                          struct instruction_data *data __rte_unused)
1858 {
1859         struct header *h;
1860
1861         CHECK(n_tokens == 2, EINVAL);
1862
1863         h = header_parse(p, tokens[1]);
1864         CHECK(h, EINVAL);
1865
1866         instr->type = INSTR_HDR_EMIT;
1867         instr->io.hdr.header_id[0] = h->id;
1868         instr->io.hdr.struct_id[0] = h->struct_id;
1869         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1870         return 0;
1871 }
1872
1873 static inline void
1874 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1875 {
1876         struct thread *t = &p->threads[p->thread_id];
1877         struct instruction *ip = t->ip;
1878
1879         __instr_hdr_emit_exec(p, t, ip);
1880
1881         /* Thread. */
1882         thread_ip_inc(p);
1883 }
1884
1885 static inline void
1886 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1887 {
1888         struct thread *t = &p->threads[p->thread_id];
1889         struct instruction *ip = t->ip;
1890
1891         __instr_hdr_emit_tx_exec(p, t, ip);
1892
1893         /* Thread. */
1894         thread_ip_reset(p, t);
1895         instr_rx_exec(p);
1896 }
1897
1898 static inline void
1899 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1900 {
1901         struct thread *t = &p->threads[p->thread_id];
1902         struct instruction *ip = t->ip;
1903
1904         __instr_hdr_emit2_tx_exec(p, t, ip);
1905
1906         /* Thread. */
1907         thread_ip_reset(p, t);
1908         instr_rx_exec(p);
1909 }
1910
1911 static inline void
1912 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1913 {
1914         struct thread *t = &p->threads[p->thread_id];
1915         struct instruction *ip = t->ip;
1916
1917         __instr_hdr_emit3_tx_exec(p, t, ip);
1918
1919         /* Thread. */
1920         thread_ip_reset(p, t);
1921         instr_rx_exec(p);
1922 }
1923
1924 static inline void
1925 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1926 {
1927         struct thread *t = &p->threads[p->thread_id];
1928         struct instruction *ip = t->ip;
1929
1930         __instr_hdr_emit4_tx_exec(p, t, ip);
1931
1932         /* Thread. */
1933         thread_ip_reset(p, t);
1934         instr_rx_exec(p);
1935 }
1936
1937 static inline void
1938 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1939 {
1940         struct thread *t = &p->threads[p->thread_id];
1941         struct instruction *ip = t->ip;
1942
1943         __instr_hdr_emit5_tx_exec(p, t, ip);
1944
1945         /* Thread. */
1946         thread_ip_reset(p, t);
1947         instr_rx_exec(p);
1948 }
1949
1950 static inline void
1951 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1952 {
1953         struct thread *t = &p->threads[p->thread_id];
1954         struct instruction *ip = t->ip;
1955
1956         __instr_hdr_emit6_tx_exec(p, t, ip);
1957
1958         /* Thread. */
1959         thread_ip_reset(p, t);
1960         instr_rx_exec(p);
1961 }
1962
1963 static inline void
1964 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1965 {
1966         struct thread *t = &p->threads[p->thread_id];
1967         struct instruction *ip = t->ip;
1968
1969         __instr_hdr_emit7_tx_exec(p, t, ip);
1970
1971         /* Thread. */
1972         thread_ip_reset(p, t);
1973         instr_rx_exec(p);
1974 }
1975
1976 static inline void
1977 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1978 {
1979         struct thread *t = &p->threads[p->thread_id];
1980         struct instruction *ip = t->ip;
1981
1982         __instr_hdr_emit8_tx_exec(p, t, ip);
1983
1984         /* Thread. */
1985         thread_ip_reset(p, t);
1986         instr_rx_exec(p);
1987 }
1988
1989 /*
1990  * validate.
1991  */
1992 static int
1993 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1994                              struct action *action __rte_unused,
1995                              char **tokens,
1996                              int n_tokens,
1997                              struct instruction *instr,
1998                              struct instruction_data *data __rte_unused)
1999 {
2000         struct header *h;
2001
2002         CHECK(n_tokens == 2, EINVAL);
2003
2004         h = header_parse(p, tokens[1]);
2005         CHECK(h, EINVAL);
2006
2007         instr->type = INSTR_HDR_VALIDATE;
2008         instr->valid.header_id = h->id;
2009         return 0;
2010 }
2011
2012 static inline void
2013 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2014 {
2015         struct thread *t = &p->threads[p->thread_id];
2016         struct instruction *ip = t->ip;
2017
2018         __instr_hdr_validate_exec(p, t, ip);
2019
2020         /* Thread. */
2021         thread_ip_inc(p);
2022 }
2023
2024 /*
2025  * invalidate.
2026  */
2027 static int
2028 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2029                                struct action *action __rte_unused,
2030                                char **tokens,
2031                                int n_tokens,
2032                                struct instruction *instr,
2033                                struct instruction_data *data __rte_unused)
2034 {
2035         struct header *h;
2036
2037         CHECK(n_tokens == 2, EINVAL);
2038
2039         h = header_parse(p, tokens[1]);
2040         CHECK(h, EINVAL);
2041
2042         instr->type = INSTR_HDR_INVALIDATE;
2043         instr->valid.header_id = h->id;
2044         return 0;
2045 }
2046
2047 static inline void
2048 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2049 {
2050         struct thread *t = &p->threads[p->thread_id];
2051         struct instruction *ip = t->ip;
2052
2053         __instr_hdr_invalidate_exec(p, t, ip);
2054
2055         /* Thread. */
2056         thread_ip_inc(p);
2057 }
2058
2059 /*
2060  * table.
2061  */
2062 static struct table *
2063 table_find(struct rte_swx_pipeline *p, const char *name);
2064
2065 static struct selector *
2066 selector_find(struct rte_swx_pipeline *p, const char *name);
2067
2068 static struct learner *
2069 learner_find(struct rte_swx_pipeline *p, const char *name);
2070
2071 static int
2072 instr_table_translate(struct rte_swx_pipeline *p,
2073                       struct action *action,
2074                       char **tokens,
2075                       int n_tokens,
2076                       struct instruction *instr,
2077                       struct instruction_data *data __rte_unused)
2078 {
2079         struct table *t;
2080         struct selector *s;
2081         struct learner *l;
2082
2083         CHECK(!action, EINVAL);
2084         CHECK(n_tokens == 2, EINVAL);
2085
2086         t = table_find(p, tokens[1]);
2087         if (t) {
2088                 instr->type = INSTR_TABLE;
2089                 instr->table.table_id = t->id;
2090                 return 0;
2091         }
2092
2093         s = selector_find(p, tokens[1]);
2094         if (s) {
2095                 instr->type = INSTR_SELECTOR;
2096                 instr->table.table_id = s->id;
2097                 return 0;
2098         }
2099
2100         l = learner_find(p, tokens[1]);
2101         if (l) {
2102                 instr->type = INSTR_LEARNER;
2103                 instr->table.table_id = l->id;
2104                 return 0;
2105         }
2106
2107         CHECK(0, EINVAL);
2108 }
2109
2110 static inline void
2111 instr_table_exec(struct rte_swx_pipeline *p)
2112 {
2113         struct thread *t = &p->threads[p->thread_id];
2114         struct instruction *ip = t->ip;
2115         uint32_t table_id = ip->table.table_id;
2116         struct rte_swx_table_state *ts = &t->table_state[table_id];
2117         struct table_runtime *table = &t->tables[table_id];
2118         struct table_statistics *stats = &p->table_stats[table_id];
2119         uint64_t action_id, n_pkts_hit, n_pkts_action;
2120         uint8_t *action_data;
2121         int done, hit;
2122
2123         /* Table. */
2124         done = table->func(ts->obj,
2125                            table->mailbox,
2126                            table->key,
2127                            &action_id,
2128                            &action_data,
2129                            &hit);
2130         if (!done) {
2131                 /* Thread. */
2132                 TRACE("[Thread %2u] table %u (not finalized)\n",
2133                       p->thread_id,
2134                       table_id);
2135
2136                 thread_yield(p);
2137                 return;
2138         }
2139
2140         action_id = hit ? action_id : ts->default_action_id;
2141         action_data = hit ? action_data : ts->default_action_data;
2142         n_pkts_hit = stats->n_pkts_hit[hit];
2143         n_pkts_action = stats->n_pkts_action[action_id];
2144
2145         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2146               p->thread_id,
2147               table_id,
2148               hit ? "hit" : "miss",
2149               (uint32_t)action_id);
2150
2151         t->action_id = action_id;
2152         t->structs[0] = action_data;
2153         t->hit = hit;
2154         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2155         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2156
2157         /* Thread. */
2158         thread_ip_action_call(p, t, action_id);
2159 }
2160
2161 static inline void
2162 instr_table_af_exec(struct rte_swx_pipeline *p)
2163 {
2164         struct thread *t = &p->threads[p->thread_id];
2165         struct instruction *ip = t->ip;
2166         uint32_t table_id = ip->table.table_id;
2167         struct rte_swx_table_state *ts = &t->table_state[table_id];
2168         struct table_runtime *table = &t->tables[table_id];
2169         struct table_statistics *stats = &p->table_stats[table_id];
2170         uint64_t action_id, n_pkts_hit, n_pkts_action;
2171         uint8_t *action_data;
2172         action_func_t action_func;
2173         int done, hit;
2174
2175         /* Table. */
2176         done = table->func(ts->obj,
2177                            table->mailbox,
2178                            table->key,
2179                            &action_id,
2180                            &action_data,
2181                            &hit);
2182         if (!done) {
2183                 /* Thread. */
2184                 TRACE("[Thread %2u] table %u (not finalized)\n",
2185                       p->thread_id,
2186                       table_id);
2187
2188                 thread_yield(p);
2189                 return;
2190         }
2191
2192         action_id = hit ? action_id : ts->default_action_id;
2193         action_data = hit ? action_data : ts->default_action_data;
2194         action_func = p->action_funcs[action_id];
2195         n_pkts_hit = stats->n_pkts_hit[hit];
2196         n_pkts_action = stats->n_pkts_action[action_id];
2197
2198         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2199               p->thread_id,
2200               table_id,
2201               hit ? "hit" : "miss",
2202               (uint32_t)action_id);
2203
2204         t->action_id = action_id;
2205         t->structs[0] = action_data;
2206         t->hit = hit;
2207         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2208         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2209
2210         /* Thread. */
2211         thread_ip_inc(p);
2212
2213         /* Action. */
2214         action_func(p);
2215 }
2216
2217 static inline void
2218 instr_selector_exec(struct rte_swx_pipeline *p)
2219 {
2220         struct thread *t = &p->threads[p->thread_id];
2221         struct instruction *ip = t->ip;
2222         uint32_t selector_id = ip->table.table_id;
2223         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2224         struct selector_runtime *selector = &t->selectors[selector_id];
2225         struct selector_statistics *stats = &p->selector_stats[selector_id];
2226         uint64_t n_pkts = stats->n_pkts;
2227         int done;
2228
2229         /* Table. */
2230         done = rte_swx_table_selector_select(ts->obj,
2231                            selector->mailbox,
2232                            selector->group_id_buffer,
2233                            selector->selector_buffer,
2234                            selector->member_id_buffer);
2235         if (!done) {
2236                 /* Thread. */
2237                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2238                       p->thread_id,
2239                       selector_id);
2240
2241                 thread_yield(p);
2242                 return;
2243         }
2244
2245
2246         TRACE("[Thread %2u] selector %u\n",
2247               p->thread_id,
2248               selector_id);
2249
2250         stats->n_pkts = n_pkts + 1;
2251
2252         /* Thread. */
2253         thread_ip_inc(p);
2254 }
2255
2256 static inline void
2257 instr_learner_exec(struct rte_swx_pipeline *p)
2258 {
2259         struct thread *t = &p->threads[p->thread_id];
2260         struct instruction *ip = t->ip;
2261         uint32_t learner_id = ip->table.table_id;
2262         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2263                 p->n_selectors + learner_id];
2264         struct learner_runtime *l = &t->learners[learner_id];
2265         struct learner_statistics *stats = &p->learner_stats[learner_id];
2266         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2267         uint8_t *action_data;
2268         int done, hit;
2269
2270         /* Table. */
2271         time = rte_get_tsc_cycles();
2272
2273         done = rte_swx_table_learner_lookup(ts->obj,
2274                                             l->mailbox,
2275                                             time,
2276                                             l->key,
2277                                             &action_id,
2278                                             &action_data,
2279                                             &hit);
2280         if (!done) {
2281                 /* Thread. */
2282                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2283                       p->thread_id,
2284                       learner_id);
2285
2286                 thread_yield(p);
2287                 return;
2288         }
2289
2290         action_id = hit ? action_id : ts->default_action_id;
2291         action_data = hit ? action_data : ts->default_action_data;
2292         n_pkts_hit = stats->n_pkts_hit[hit];
2293         n_pkts_action = stats->n_pkts_action[action_id];
2294
2295         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2296               p->thread_id,
2297               learner_id,
2298               hit ? "hit" : "miss",
2299               (uint32_t)action_id);
2300
2301         t->action_id = action_id;
2302         t->structs[0] = action_data;
2303         t->hit = hit;
2304         t->learner_id = learner_id;
2305         t->time = time;
2306         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2307         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2308
2309         /* Thread. */
2310         thread_ip_action_call(p, t, action_id);
2311 }
2312
2313 static inline void
2314 instr_learner_af_exec(struct rte_swx_pipeline *p)
2315 {
2316         struct thread *t = &p->threads[p->thread_id];
2317         struct instruction *ip = t->ip;
2318         uint32_t learner_id = ip->table.table_id;
2319         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2320                 p->n_selectors + learner_id];
2321         struct learner_runtime *l = &t->learners[learner_id];
2322         struct learner_statistics *stats = &p->learner_stats[learner_id];
2323         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2324         uint8_t *action_data;
2325         action_func_t action_func;
2326         int done, hit;
2327
2328         /* Table. */
2329         time = rte_get_tsc_cycles();
2330
2331         done = rte_swx_table_learner_lookup(ts->obj,
2332                                             l->mailbox,
2333                                             time,
2334                                             l->key,
2335                                             &action_id,
2336                                             &action_data,
2337                                             &hit);
2338         if (!done) {
2339                 /* Thread. */
2340                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2341                       p->thread_id,
2342                       learner_id);
2343
2344                 thread_yield(p);
2345                 return;
2346         }
2347
2348         action_id = hit ? action_id : ts->default_action_id;
2349         action_data = hit ? action_data : ts->default_action_data;
2350         action_func = p->action_funcs[action_id];
2351         n_pkts_hit = stats->n_pkts_hit[hit];
2352         n_pkts_action = stats->n_pkts_action[action_id];
2353
2354         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2355               p->thread_id,
2356               learner_id,
2357               hit ? "hit" : "miss",
2358               (uint32_t)action_id);
2359
2360         t->action_id = action_id;
2361         t->structs[0] = action_data;
2362         t->hit = hit;
2363         t->learner_id = learner_id;
2364         t->time = time;
2365         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2366         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2367
2368         /* Thread. */
2369         thread_ip_action_call(p, t, action_id);
2370
2371         /* Action */
2372         action_func(p);
2373 }
2374
2375 /*
2376  * learn.
2377  */
2378 static struct action *
2379 action_find(struct rte_swx_pipeline *p, const char *name);
2380
2381 static int
2382 action_has_nbo_args(struct action *a);
2383
2384 static int
2385 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2386
2387 static int
2388 instr_learn_translate(struct rte_swx_pipeline *p,
2389                       struct action *action,
2390                       char **tokens,
2391                       int n_tokens,
2392                       struct instruction *instr,
2393                       struct instruction_data *data __rte_unused)
2394 {
2395         struct action *a;
2396         const char *mf_name;
2397         uint32_t mf_offset = 0;
2398
2399         CHECK(action, EINVAL);
2400         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2401
2402         a = action_find(p, tokens[1]);
2403         CHECK(a, EINVAL);
2404         CHECK(!action_has_nbo_args(a), EINVAL);
2405
2406         mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2407         CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2408
2409         if (mf_name) {
2410                 struct field *mf;
2411
2412                 mf = metadata_field_parse(p, mf_name);
2413                 CHECK(mf, EINVAL);
2414
2415                 mf_offset = mf->offset / 8;
2416         }
2417
2418         instr->type = INSTR_LEARNER_LEARN;
2419         instr->learn.action_id = a->id;
2420         instr->learn.mf_offset = mf_offset;
2421
2422         return 0;
2423 }
2424
2425 static inline void
2426 instr_learn_exec(struct rte_swx_pipeline *p)
2427 {
2428         struct thread *t = &p->threads[p->thread_id];
2429         struct instruction *ip = t->ip;
2430
2431         __instr_learn_exec(p, t, ip);
2432
2433         /* Thread. */
2434         thread_ip_inc(p);
2435 }
2436
2437 /*
2438  * forget.
2439  */
2440 static int
2441 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2442                        struct action *action,
2443                        char **tokens __rte_unused,
2444                        int n_tokens,
2445                        struct instruction *instr,
2446                        struct instruction_data *data __rte_unused)
2447 {
2448         CHECK(action, EINVAL);
2449         CHECK(n_tokens == 1, EINVAL);
2450
2451         instr->type = INSTR_LEARNER_FORGET;
2452
2453         return 0;
2454 }
2455
2456 static inline void
2457 instr_forget_exec(struct rte_swx_pipeline *p)
2458 {
2459         struct thread *t = &p->threads[p->thread_id];
2460         struct instruction *ip = t->ip;
2461
2462         __instr_forget_exec(p, t, ip);
2463
2464         /* Thread. */
2465         thread_ip_inc(p);
2466 }
2467
2468 /*
2469  * extern.
2470  */
2471 static int
2472 instr_extern_translate(struct rte_swx_pipeline *p,
2473                        struct action *action __rte_unused,
2474                        char **tokens,
2475                        int n_tokens,
2476                        struct instruction *instr,
2477                        struct instruction_data *data __rte_unused)
2478 {
2479         char *token = tokens[1];
2480
2481         CHECK(n_tokens == 2, EINVAL);
2482
2483         if (token[0] == 'e') {
2484                 struct extern_obj *obj;
2485                 struct extern_type_member_func *func;
2486
2487                 func = extern_obj_member_func_parse(p, token, &obj);
2488                 CHECK(func, EINVAL);
2489
2490                 instr->type = INSTR_EXTERN_OBJ;
2491                 instr->ext_obj.ext_obj_id = obj->id;
2492                 instr->ext_obj.func_id = func->id;
2493
2494                 return 0;
2495         }
2496
2497         if (token[0] == 'f') {
2498                 struct extern_func *func;
2499
2500                 func = extern_func_parse(p, token);
2501                 CHECK(func, EINVAL);
2502
2503                 instr->type = INSTR_EXTERN_FUNC;
2504                 instr->ext_func.ext_func_id = func->id;
2505
2506                 return 0;
2507         }
2508
2509         CHECK(0, EINVAL);
2510 }
2511
2512 static inline void
2513 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2514 {
2515         struct thread *t = &p->threads[p->thread_id];
2516         struct instruction *ip = t->ip;
2517         uint32_t done;
2518
2519         /* Extern object member function execute. */
2520         done = __instr_extern_obj_exec(p, t, ip);
2521
2522         /* Thread. */
2523         thread_ip_inc_cond(t, done);
2524         thread_yield_cond(p, done ^ 1);
2525 }
2526
2527 static inline void
2528 instr_extern_func_exec(struct rte_swx_pipeline *p)
2529 {
2530         struct thread *t = &p->threads[p->thread_id];
2531         struct instruction *ip = t->ip;
2532         uint32_t done;
2533
2534         /* Extern function execute. */
2535         done = __instr_extern_func_exec(p, t, ip);
2536
2537         /* Thread. */
2538         thread_ip_inc_cond(t, done);
2539         thread_yield_cond(p, done ^ 1);
2540 }
2541
2542 /*
2543  * mov.
2544  */
2545 static int
2546 instr_mov_translate(struct rte_swx_pipeline *p,
2547                     struct action *action,
2548                     char **tokens,
2549                     int n_tokens,
2550                     struct instruction *instr,
2551                     struct instruction_data *data __rte_unused)
2552 {
2553         char *dst = tokens[1], *src = tokens[2];
2554         struct field *fdst, *fsrc;
2555         uint64_t src_val;
2556         uint32_t dst_struct_id = 0, src_struct_id = 0;
2557
2558         CHECK(n_tokens == 3, EINVAL);
2559
2560         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2561         CHECK(fdst, EINVAL);
2562         CHECK(!fdst->var_size, EINVAL);
2563
2564         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2565         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2566         if (fsrc) {
2567                 CHECK(!fsrc->var_size, EINVAL);
2568
2569                 instr->type = INSTR_MOV;
2570                 if (dst[0] != 'h' && src[0] == 'h')
2571                         instr->type = INSTR_MOV_MH;
2572                 if (dst[0] == 'h' && src[0] != 'h')
2573                         instr->type = INSTR_MOV_HM;
2574                 if (dst[0] == 'h' && src[0] == 'h')
2575                         instr->type = INSTR_MOV_HH;
2576
2577                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2578                 instr->mov.dst.n_bits = fdst->n_bits;
2579                 instr->mov.dst.offset = fdst->offset / 8;
2580                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2581                 instr->mov.src.n_bits = fsrc->n_bits;
2582                 instr->mov.src.offset = fsrc->offset / 8;
2583                 return 0;
2584         }
2585
2586         /* MOV_I. */
2587         src_val = strtoull(src, &src, 0);
2588         CHECK(!src[0], EINVAL);
2589
2590         if (dst[0] == 'h')
2591                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2592
2593         instr->type = INSTR_MOV_I;
2594         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2595         instr->mov.dst.n_bits = fdst->n_bits;
2596         instr->mov.dst.offset = fdst->offset / 8;
2597         instr->mov.src_val = src_val;
2598         return 0;
2599 }
2600
2601 static inline void
2602 instr_mov_exec(struct rte_swx_pipeline *p)
2603 {
2604         struct thread *t = &p->threads[p->thread_id];
2605         struct instruction *ip = t->ip;
2606
2607         __instr_mov_exec(p, t, ip);
2608
2609         /* Thread. */
2610         thread_ip_inc(p);
2611 }
2612
2613 static inline void
2614 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2615 {
2616         struct thread *t = &p->threads[p->thread_id];
2617         struct instruction *ip = t->ip;
2618
2619         __instr_mov_mh_exec(p, t, ip);
2620
2621         /* Thread. */
2622         thread_ip_inc(p);
2623 }
2624
2625 static inline void
2626 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2627 {
2628         struct thread *t = &p->threads[p->thread_id];
2629         struct instruction *ip = t->ip;
2630
2631         __instr_mov_hm_exec(p, t, ip);
2632
2633         /* Thread. */
2634         thread_ip_inc(p);
2635 }
2636
2637 static inline void
2638 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2639 {
2640         struct thread *t = &p->threads[p->thread_id];
2641         struct instruction *ip = t->ip;
2642
2643         __instr_mov_hh_exec(p, t, ip);
2644
2645         /* Thread. */
2646         thread_ip_inc(p);
2647 }
2648
2649 static inline void
2650 instr_mov_i_exec(struct rte_swx_pipeline *p)
2651 {
2652         struct thread *t = &p->threads[p->thread_id];
2653         struct instruction *ip = t->ip;
2654
2655         __instr_mov_i_exec(p, t, ip);
2656
2657         /* Thread. */
2658         thread_ip_inc(p);
2659 }
2660
2661 /*
2662  * dma.
2663  */
2664 static inline void
2665 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2666 {
2667         struct thread *t = &p->threads[p->thread_id];
2668         struct instruction *ip = t->ip;
2669
2670         __instr_dma_ht_exec(p, t, ip);
2671
2672         /* Thread. */
2673         thread_ip_inc(p);
2674 }
2675
2676 static inline void
2677 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2678 {
2679         struct thread *t = &p->threads[p->thread_id];
2680         struct instruction *ip = t->ip;
2681
2682         __instr_dma_ht2_exec(p, t, ip);
2683
2684         /* Thread. */
2685         thread_ip_inc(p);
2686 }
2687
2688 static inline void
2689 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2690 {
2691         struct thread *t = &p->threads[p->thread_id];
2692         struct instruction *ip = t->ip;
2693
2694         __instr_dma_ht3_exec(p, t, ip);
2695
2696         /* Thread. */
2697         thread_ip_inc(p);
2698 }
2699
2700 static inline void
2701 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2702 {
2703         struct thread *t = &p->threads[p->thread_id];
2704         struct instruction *ip = t->ip;
2705
2706         __instr_dma_ht4_exec(p, t, ip);
2707
2708         /* Thread. */
2709         thread_ip_inc(p);
2710 }
2711
2712 static inline void
2713 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2714 {
2715         struct thread *t = &p->threads[p->thread_id];
2716         struct instruction *ip = t->ip;
2717
2718         __instr_dma_ht5_exec(p, t, ip);
2719
2720         /* Thread. */
2721         thread_ip_inc(p);
2722 }
2723
2724 static inline void
2725 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2726 {
2727         struct thread *t = &p->threads[p->thread_id];
2728         struct instruction *ip = t->ip;
2729
2730         __instr_dma_ht6_exec(p, t, ip);
2731
2732         /* Thread. */
2733         thread_ip_inc(p);
2734 }
2735
2736 static inline void
2737 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2738 {
2739         struct thread *t = &p->threads[p->thread_id];
2740         struct instruction *ip = t->ip;
2741
2742         __instr_dma_ht7_exec(p, t, ip);
2743
2744         /* Thread. */
2745         thread_ip_inc(p);
2746 }
2747
2748 static inline void
2749 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2750 {
2751         struct thread *t = &p->threads[p->thread_id];
2752         struct instruction *ip = t->ip;
2753
2754         __instr_dma_ht8_exec(p, t, ip);
2755
2756         /* Thread. */
2757         thread_ip_inc(p);
2758 }
2759
2760 /*
2761  * alu.
2762  */
2763 static int
2764 instr_alu_add_translate(struct rte_swx_pipeline *p,
2765                         struct action *action,
2766                         char **tokens,
2767                         int n_tokens,
2768                         struct instruction *instr,
2769                         struct instruction_data *data __rte_unused)
2770 {
2771         char *dst = tokens[1], *src = tokens[2];
2772         struct field *fdst, *fsrc;
2773         uint64_t src_val;
2774         uint32_t dst_struct_id = 0, src_struct_id = 0;
2775
2776         CHECK(n_tokens == 3, EINVAL);
2777
2778         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2779         CHECK(fdst, EINVAL);
2780         CHECK(!fdst->var_size, EINVAL);
2781
2782         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2783         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2784         if (fsrc) {
2785                 CHECK(!fsrc->var_size, EINVAL);
2786
2787                 instr->type = INSTR_ALU_ADD;
2788                 if (dst[0] == 'h' && src[0] != 'h')
2789                         instr->type = INSTR_ALU_ADD_HM;
2790                 if (dst[0] != 'h' && src[0] == 'h')
2791                         instr->type = INSTR_ALU_ADD_MH;
2792                 if (dst[0] == 'h' && src[0] == 'h')
2793                         instr->type = INSTR_ALU_ADD_HH;
2794
2795                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2796                 instr->alu.dst.n_bits = fdst->n_bits;
2797                 instr->alu.dst.offset = fdst->offset / 8;
2798                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2799                 instr->alu.src.n_bits = fsrc->n_bits;
2800                 instr->alu.src.offset = fsrc->offset / 8;
2801                 return 0;
2802         }
2803
2804         /* ADD_MI, ADD_HI. */
2805         src_val = strtoull(src, &src, 0);
2806         CHECK(!src[0], EINVAL);
2807
2808         instr->type = INSTR_ALU_ADD_MI;
2809         if (dst[0] == 'h')
2810                 instr->type = INSTR_ALU_ADD_HI;
2811
2812         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2813         instr->alu.dst.n_bits = fdst->n_bits;
2814         instr->alu.dst.offset = fdst->offset / 8;
2815         instr->alu.src_val = src_val;
2816         return 0;
2817 }
2818
2819 static int
2820 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2821                         struct action *action,
2822                         char **tokens,
2823                         int n_tokens,
2824                         struct instruction *instr,
2825                         struct instruction_data *data __rte_unused)
2826 {
2827         char *dst = tokens[1], *src = tokens[2];
2828         struct field *fdst, *fsrc;
2829         uint64_t src_val;
2830         uint32_t dst_struct_id = 0, src_struct_id = 0;
2831
2832         CHECK(n_tokens == 3, EINVAL);
2833
2834         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2835         CHECK(fdst, EINVAL);
2836         CHECK(!fdst->var_size, EINVAL);
2837
2838         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2839         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2840         if (fsrc) {
2841                 CHECK(!fsrc->var_size, EINVAL);
2842
2843                 instr->type = INSTR_ALU_SUB;
2844                 if (dst[0] == 'h' && src[0] != 'h')
2845                         instr->type = INSTR_ALU_SUB_HM;
2846                 if (dst[0] != 'h' && src[0] == 'h')
2847                         instr->type = INSTR_ALU_SUB_MH;
2848                 if (dst[0] == 'h' && src[0] == 'h')
2849                         instr->type = INSTR_ALU_SUB_HH;
2850
2851                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2852                 instr->alu.dst.n_bits = fdst->n_bits;
2853                 instr->alu.dst.offset = fdst->offset / 8;
2854                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2855                 instr->alu.src.n_bits = fsrc->n_bits;
2856                 instr->alu.src.offset = fsrc->offset / 8;
2857                 return 0;
2858         }
2859
2860         /* SUB_MI, SUB_HI. */
2861         src_val = strtoull(src, &src, 0);
2862         CHECK(!src[0], EINVAL);
2863
2864         instr->type = INSTR_ALU_SUB_MI;
2865         if (dst[0] == 'h')
2866                 instr->type = INSTR_ALU_SUB_HI;
2867
2868         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2869         instr->alu.dst.n_bits = fdst->n_bits;
2870         instr->alu.dst.offset = fdst->offset / 8;
2871         instr->alu.src_val = src_val;
2872         return 0;
2873 }
2874
2875 static int
2876 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2877                           struct action *action __rte_unused,
2878                           char **tokens,
2879                           int n_tokens,
2880                           struct instruction *instr,
2881                           struct instruction_data *data __rte_unused)
2882 {
2883         char *dst = tokens[1], *src = tokens[2];
2884         struct header *hdst, *hsrc;
2885         struct field *fdst, *fsrc;
2886
2887         CHECK(n_tokens == 3, EINVAL);
2888
2889         fdst = header_field_parse(p, dst, &hdst);
2890         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2891         CHECK(!fdst->var_size, EINVAL);
2892
2893         /* CKADD_FIELD. */
2894         fsrc = header_field_parse(p, src, &hsrc);
2895         if (fsrc) {
2896                 CHECK(!fsrc->var_size, EINVAL);
2897
2898                 instr->type = INSTR_ALU_CKADD_FIELD;
2899                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2900                 instr->alu.dst.n_bits = fdst->n_bits;
2901                 instr->alu.dst.offset = fdst->offset / 8;
2902                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2903                 instr->alu.src.n_bits = fsrc->n_bits;
2904                 instr->alu.src.offset = fsrc->offset / 8;
2905                 return 0;
2906         }
2907
2908         /* CKADD_STRUCT, CKADD_STRUCT20. */
2909         hsrc = header_parse(p, src);
2910         CHECK(hsrc, EINVAL);
2911         CHECK(!hsrc->st->var_size, EINVAL);
2912
2913         instr->type = INSTR_ALU_CKADD_STRUCT;
2914         if ((hsrc->st->n_bits / 8) == 20)
2915                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2916
2917         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2918         instr->alu.dst.n_bits = fdst->n_bits;
2919         instr->alu.dst.offset = fdst->offset / 8;
2920         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2921         instr->alu.src.n_bits = hsrc->st->n_bits;
2922         instr->alu.src.offset = 0; /* Unused. */
2923         return 0;
2924 }
2925
2926 static int
2927 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2928                           struct action *action __rte_unused,
2929                           char **tokens,
2930                           int n_tokens,
2931                           struct instruction *instr,
2932                           struct instruction_data *data __rte_unused)
2933 {
2934         char *dst = tokens[1], *src = tokens[2];
2935         struct header *hdst, *hsrc;
2936         struct field *fdst, *fsrc;
2937
2938         CHECK(n_tokens == 3, EINVAL);
2939
2940         fdst = header_field_parse(p, dst, &hdst);
2941         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2942         CHECK(!fdst->var_size, EINVAL);
2943
2944         fsrc = header_field_parse(p, src, &hsrc);
2945         CHECK(fsrc, EINVAL);
2946         CHECK(!fsrc->var_size, EINVAL);
2947
2948         instr->type = INSTR_ALU_CKSUB_FIELD;
2949         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2950         instr->alu.dst.n_bits = fdst->n_bits;
2951         instr->alu.dst.offset = fdst->offset / 8;
2952         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2953         instr->alu.src.n_bits = fsrc->n_bits;
2954         instr->alu.src.offset = fsrc->offset / 8;
2955         return 0;
2956 }
2957
2958 static int
2959 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2960                         struct action *action,
2961                         char **tokens,
2962                         int n_tokens,
2963                         struct instruction *instr,
2964                         struct instruction_data *data __rte_unused)
2965 {
2966         char *dst = tokens[1], *src = tokens[2];
2967         struct field *fdst, *fsrc;
2968         uint64_t src_val;
2969         uint32_t dst_struct_id = 0, src_struct_id = 0;
2970
2971         CHECK(n_tokens == 3, EINVAL);
2972
2973         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2974         CHECK(fdst, EINVAL);
2975         CHECK(!fdst->var_size, EINVAL);
2976
2977         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2978         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2979         if (fsrc) {
2980                 CHECK(!fsrc->var_size, EINVAL);
2981
2982                 instr->type = INSTR_ALU_SHL;
2983                 if (dst[0] == 'h' && src[0] != 'h')
2984                         instr->type = INSTR_ALU_SHL_HM;
2985                 if (dst[0] != 'h' && src[0] == 'h')
2986                         instr->type = INSTR_ALU_SHL_MH;
2987                 if (dst[0] == 'h' && src[0] == 'h')
2988                         instr->type = INSTR_ALU_SHL_HH;
2989
2990                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2991                 instr->alu.dst.n_bits = fdst->n_bits;
2992                 instr->alu.dst.offset = fdst->offset / 8;
2993                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2994                 instr->alu.src.n_bits = fsrc->n_bits;
2995                 instr->alu.src.offset = fsrc->offset / 8;
2996                 return 0;
2997         }
2998
2999         /* SHL_MI, SHL_HI. */
3000         src_val = strtoull(src, &src, 0);
3001         CHECK(!src[0], EINVAL);
3002
3003         instr->type = INSTR_ALU_SHL_MI;
3004         if (dst[0] == 'h')
3005                 instr->type = INSTR_ALU_SHL_HI;
3006
3007         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3008         instr->alu.dst.n_bits = fdst->n_bits;
3009         instr->alu.dst.offset = fdst->offset / 8;
3010         instr->alu.src_val = src_val;
3011         return 0;
3012 }
3013
3014 static int
3015 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3016                         struct action *action,
3017                         char **tokens,
3018                         int n_tokens,
3019                         struct instruction *instr,
3020                         struct instruction_data *data __rte_unused)
3021 {
3022         char *dst = tokens[1], *src = tokens[2];
3023         struct field *fdst, *fsrc;
3024         uint64_t src_val;
3025         uint32_t dst_struct_id = 0, src_struct_id = 0;
3026
3027         CHECK(n_tokens == 3, EINVAL);
3028
3029         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3030         CHECK(fdst, EINVAL);
3031         CHECK(!fdst->var_size, EINVAL);
3032
3033         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3034         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3035         if (fsrc) {
3036                 CHECK(!fsrc->var_size, EINVAL);
3037
3038                 instr->type = INSTR_ALU_SHR;
3039                 if (dst[0] == 'h' && src[0] != 'h')
3040                         instr->type = INSTR_ALU_SHR_HM;
3041                 if (dst[0] != 'h' && src[0] == 'h')
3042                         instr->type = INSTR_ALU_SHR_MH;
3043                 if (dst[0] == 'h' && src[0] == 'h')
3044                         instr->type = INSTR_ALU_SHR_HH;
3045
3046                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3047                 instr->alu.dst.n_bits = fdst->n_bits;
3048                 instr->alu.dst.offset = fdst->offset / 8;
3049                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3050                 instr->alu.src.n_bits = fsrc->n_bits;
3051                 instr->alu.src.offset = fsrc->offset / 8;
3052                 return 0;
3053         }
3054
3055         /* SHR_MI, SHR_HI. */
3056         src_val = strtoull(src, &src, 0);
3057         CHECK(!src[0], EINVAL);
3058
3059         instr->type = INSTR_ALU_SHR_MI;
3060         if (dst[0] == 'h')
3061                 instr->type = INSTR_ALU_SHR_HI;
3062
3063         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3064         instr->alu.dst.n_bits = fdst->n_bits;
3065         instr->alu.dst.offset = fdst->offset / 8;
3066         instr->alu.src_val = src_val;
3067         return 0;
3068 }
3069
3070 static int
3071 instr_alu_and_translate(struct rte_swx_pipeline *p,
3072                         struct action *action,
3073                         char **tokens,
3074                         int n_tokens,
3075                         struct instruction *instr,
3076                         struct instruction_data *data __rte_unused)
3077 {
3078         char *dst = tokens[1], *src = tokens[2];
3079         struct field *fdst, *fsrc;
3080         uint64_t src_val;
3081         uint32_t dst_struct_id = 0, src_struct_id = 0;
3082
3083         CHECK(n_tokens == 3, EINVAL);
3084
3085         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3086         CHECK(fdst, EINVAL);
3087         CHECK(!fdst->var_size, EINVAL);
3088
3089         /* AND, AND_MH, AND_HM, AND_HH. */
3090         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3091         if (fsrc) {
3092                 CHECK(!fsrc->var_size, EINVAL);
3093
3094                 instr->type = INSTR_ALU_AND;
3095                 if (dst[0] != 'h' && src[0] == 'h')
3096                         instr->type = INSTR_ALU_AND_MH;
3097                 if (dst[0] == 'h' && src[0] != 'h')
3098                         instr->type = INSTR_ALU_AND_HM;
3099                 if (dst[0] == 'h' && src[0] == 'h')
3100                         instr->type = INSTR_ALU_AND_HH;
3101
3102                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3103                 instr->alu.dst.n_bits = fdst->n_bits;
3104                 instr->alu.dst.offset = fdst->offset / 8;
3105                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3106                 instr->alu.src.n_bits = fsrc->n_bits;
3107                 instr->alu.src.offset = fsrc->offset / 8;
3108                 return 0;
3109         }
3110
3111         /* AND_I. */
3112         src_val = strtoull(src, &src, 0);
3113         CHECK(!src[0], EINVAL);
3114
3115         if (dst[0] == 'h')
3116                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3117
3118         instr->type = INSTR_ALU_AND_I;
3119         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3120         instr->alu.dst.n_bits = fdst->n_bits;
3121         instr->alu.dst.offset = fdst->offset / 8;
3122         instr->alu.src_val = src_val;
3123         return 0;
3124 }
3125
3126 static int
3127 instr_alu_or_translate(struct rte_swx_pipeline *p,
3128                        struct action *action,
3129                        char **tokens,
3130                        int n_tokens,
3131                        struct instruction *instr,
3132                        struct instruction_data *data __rte_unused)
3133 {
3134         char *dst = tokens[1], *src = tokens[2];
3135         struct field *fdst, *fsrc;
3136         uint64_t src_val;
3137         uint32_t dst_struct_id = 0, src_struct_id = 0;
3138
3139         CHECK(n_tokens == 3, EINVAL);
3140
3141         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3142         CHECK(fdst, EINVAL);
3143         CHECK(!fdst->var_size, EINVAL);
3144
3145         /* OR, OR_MH, OR_HM, OR_HH. */
3146         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3147         if (fsrc) {
3148                 CHECK(!fsrc->var_size, EINVAL);
3149
3150                 instr->type = INSTR_ALU_OR;
3151                 if (dst[0] != 'h' && src[0] == 'h')
3152                         instr->type = INSTR_ALU_OR_MH;
3153                 if (dst[0] == 'h' && src[0] != 'h')
3154                         instr->type = INSTR_ALU_OR_HM;
3155                 if (dst[0] == 'h' && src[0] == 'h')
3156                         instr->type = INSTR_ALU_OR_HH;
3157
3158                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3159                 instr->alu.dst.n_bits = fdst->n_bits;
3160                 instr->alu.dst.offset = fdst->offset / 8;
3161                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3162                 instr->alu.src.n_bits = fsrc->n_bits;
3163                 instr->alu.src.offset = fsrc->offset / 8;
3164                 return 0;
3165         }
3166
3167         /* OR_I. */
3168         src_val = strtoull(src, &src, 0);
3169         CHECK(!src[0], EINVAL);
3170
3171         if (dst[0] == 'h')
3172                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3173
3174         instr->type = INSTR_ALU_OR_I;
3175         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3176         instr->alu.dst.n_bits = fdst->n_bits;
3177         instr->alu.dst.offset = fdst->offset / 8;
3178         instr->alu.src_val = src_val;
3179         return 0;
3180 }
3181
3182 static int
3183 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3184                         struct action *action,
3185                         char **tokens,
3186                         int n_tokens,
3187                         struct instruction *instr,
3188                         struct instruction_data *data __rte_unused)
3189 {
3190         char *dst = tokens[1], *src = tokens[2];
3191         struct field *fdst, *fsrc;
3192         uint64_t src_val;
3193         uint32_t dst_struct_id = 0, src_struct_id = 0;
3194
3195         CHECK(n_tokens == 3, EINVAL);
3196
3197         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3198         CHECK(fdst, EINVAL);
3199         CHECK(!fdst->var_size, EINVAL);
3200
3201         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3202         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3203         if (fsrc) {
3204                 CHECK(!fsrc->var_size, EINVAL);
3205
3206                 instr->type = INSTR_ALU_XOR;
3207                 if (dst[0] != 'h' && src[0] == 'h')
3208                         instr->type = INSTR_ALU_XOR_MH;
3209                 if (dst[0] == 'h' && src[0] != 'h')
3210                         instr->type = INSTR_ALU_XOR_HM;
3211                 if (dst[0] == 'h' && src[0] == 'h')
3212                         instr->type = INSTR_ALU_XOR_HH;
3213
3214                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3215                 instr->alu.dst.n_bits = fdst->n_bits;
3216                 instr->alu.dst.offset = fdst->offset / 8;
3217                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3218                 instr->alu.src.n_bits = fsrc->n_bits;
3219                 instr->alu.src.offset = fsrc->offset / 8;
3220                 return 0;
3221         }
3222
3223         /* XOR_I. */
3224         src_val = strtoull(src, &src, 0);
3225         CHECK(!src[0], EINVAL);
3226
3227         if (dst[0] == 'h')
3228                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3229
3230         instr->type = INSTR_ALU_XOR_I;
3231         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3232         instr->alu.dst.n_bits = fdst->n_bits;
3233         instr->alu.dst.offset = fdst->offset / 8;
3234         instr->alu.src_val = src_val;
3235         return 0;
3236 }
3237
3238 static inline void
3239 instr_alu_add_exec(struct rte_swx_pipeline *p)
3240 {
3241         struct thread *t = &p->threads[p->thread_id];
3242         struct instruction *ip = t->ip;
3243
3244         /* Structs */
3245         __instr_alu_add_exec(p, t, ip);
3246
3247         /* Thread. */
3248         thread_ip_inc(p);
3249 }
3250
3251 static inline void
3252 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3253 {
3254         struct thread *t = &p->threads[p->thread_id];
3255         struct instruction *ip = t->ip;
3256
3257         /* Structs. */
3258         __instr_alu_add_mh_exec(p, t, ip);
3259
3260         /* Thread. */
3261         thread_ip_inc(p);
3262 }
3263
3264 static inline void
3265 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3266 {
3267         struct thread *t = &p->threads[p->thread_id];
3268         struct instruction *ip = t->ip;
3269
3270         /* Structs. */
3271         __instr_alu_add_hm_exec(p, t, ip);
3272
3273         /* Thread. */
3274         thread_ip_inc(p);
3275 }
3276
3277 static inline void
3278 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3279 {
3280         struct thread *t = &p->threads[p->thread_id];
3281         struct instruction *ip = t->ip;
3282
3283         /* Structs. */
3284         __instr_alu_add_hh_exec(p, t, ip);
3285
3286         /* Thread. */
3287         thread_ip_inc(p);
3288 }
3289
3290 static inline void
3291 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3292 {
3293         struct thread *t = &p->threads[p->thread_id];
3294         struct instruction *ip = t->ip;
3295
3296         /* Structs. */
3297         __instr_alu_add_mi_exec(p, t, ip);
3298
3299         /* Thread. */
3300         thread_ip_inc(p);
3301 }
3302
3303 static inline void
3304 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3305 {
3306         struct thread *t = &p->threads[p->thread_id];
3307         struct instruction *ip = t->ip;
3308
3309         /* Structs. */
3310         __instr_alu_add_hi_exec(p, t, ip);
3311
3312         /* Thread. */
3313         thread_ip_inc(p);
3314 }
3315
3316 static inline void
3317 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3318 {
3319         struct thread *t = &p->threads[p->thread_id];
3320         struct instruction *ip = t->ip;
3321
3322         /* Structs. */
3323         __instr_alu_sub_exec(p, t, ip);
3324
3325         /* Thread. */
3326         thread_ip_inc(p);
3327 }
3328
3329 static inline void
3330 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3331 {
3332         struct thread *t = &p->threads[p->thread_id];
3333         struct instruction *ip = t->ip;
3334
3335         /* Structs. */
3336         __instr_alu_sub_mh_exec(p, t, ip);
3337
3338         /* Thread. */
3339         thread_ip_inc(p);
3340 }
3341
3342 static inline void
3343 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3344 {
3345         struct thread *t = &p->threads[p->thread_id];
3346         struct instruction *ip = t->ip;
3347
3348         /* Structs. */
3349         __instr_alu_sub_hm_exec(p, t, ip);
3350
3351         /* Thread. */
3352         thread_ip_inc(p);
3353 }
3354
3355 static inline void
3356 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3357 {
3358         struct thread *t = &p->threads[p->thread_id];
3359         struct instruction *ip = t->ip;
3360
3361         /* Structs. */
3362         __instr_alu_sub_hh_exec(p, t, ip);
3363
3364         /* Thread. */
3365         thread_ip_inc(p);
3366 }
3367
3368 static inline void
3369 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3370 {
3371         struct thread *t = &p->threads[p->thread_id];
3372         struct instruction *ip = t->ip;
3373
3374         /* Structs. */
3375         __instr_alu_sub_mi_exec(p, t, ip);
3376
3377         /* Thread. */
3378         thread_ip_inc(p);
3379 }
3380
3381 static inline void
3382 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3383 {
3384         struct thread *t = &p->threads[p->thread_id];
3385         struct instruction *ip = t->ip;
3386
3387         /* Structs. */
3388         __instr_alu_sub_hi_exec(p, t, ip);
3389
3390         /* Thread. */
3391         thread_ip_inc(p);
3392 }
3393
3394 static inline void
3395 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3396 {
3397         struct thread *t = &p->threads[p->thread_id];
3398         struct instruction *ip = t->ip;
3399
3400         /* Structs. */
3401         __instr_alu_shl_exec(p, t, ip);
3402
3403         /* Thread. */
3404         thread_ip_inc(p);
3405 }
3406
3407 static inline void
3408 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3409 {
3410         struct thread *t = &p->threads[p->thread_id];
3411         struct instruction *ip = t->ip;
3412
3413         /* Structs. */
3414         __instr_alu_shl_mh_exec(p, t, ip);
3415
3416         /* Thread. */
3417         thread_ip_inc(p);
3418 }
3419
3420 static inline void
3421 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3422 {
3423         struct thread *t = &p->threads[p->thread_id];
3424         struct instruction *ip = t->ip;
3425
3426         /* Structs. */
3427         __instr_alu_shl_hm_exec(p, t, ip);
3428
3429         /* Thread. */
3430         thread_ip_inc(p);
3431 }
3432
3433 static inline void
3434 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3435 {
3436         struct thread *t = &p->threads[p->thread_id];
3437         struct instruction *ip = t->ip;
3438
3439         /* Structs. */
3440         __instr_alu_shl_hh_exec(p, t, ip);
3441
3442         /* Thread. */
3443         thread_ip_inc(p);
3444 }
3445
3446 static inline void
3447 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3448 {
3449         struct thread *t = &p->threads[p->thread_id];
3450         struct instruction *ip = t->ip;
3451
3452         /* Structs. */
3453         __instr_alu_shl_mi_exec(p, t, ip);
3454
3455         /* Thread. */
3456         thread_ip_inc(p);
3457 }
3458
3459 static inline void
3460 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3461 {
3462         struct thread *t = &p->threads[p->thread_id];
3463         struct instruction *ip = t->ip;
3464
3465         /* Structs. */
3466         __instr_alu_shl_hi_exec(p, t, ip);
3467
3468         /* Thread. */
3469         thread_ip_inc(p);
3470 }
3471
3472 static inline void
3473 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3474 {
3475         struct thread *t = &p->threads[p->thread_id];
3476         struct instruction *ip = t->ip;
3477
3478         /* Structs. */
3479         __instr_alu_shr_exec(p, t, ip);
3480
3481         /* Thread. */
3482         thread_ip_inc(p);
3483 }
3484
3485 static inline void
3486 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3487 {
3488         struct thread *t = &p->threads[p->thread_id];
3489         struct instruction *ip = t->ip;
3490
3491         /* Structs. */
3492         __instr_alu_shr_mh_exec(p, t, ip);
3493
3494         /* Thread. */
3495         thread_ip_inc(p);
3496 }
3497
3498 static inline void
3499 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3500 {
3501         struct thread *t = &p->threads[p->thread_id];
3502         struct instruction *ip = t->ip;
3503
3504         /* Structs. */
3505         __instr_alu_shr_hm_exec(p, t, ip);
3506
3507         /* Thread. */
3508         thread_ip_inc(p);
3509 }
3510
3511 static inline void
3512 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3513 {
3514         struct thread *t = &p->threads[p->thread_id];
3515         struct instruction *ip = t->ip;
3516
3517         /* Structs. */
3518         __instr_alu_shr_hh_exec(p, t, ip);
3519
3520         /* Thread. */
3521         thread_ip_inc(p);
3522 }
3523
3524 static inline void
3525 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3526 {
3527         struct thread *t = &p->threads[p->thread_id];
3528         struct instruction *ip = t->ip;
3529
3530         /* Structs. */
3531         __instr_alu_shr_mi_exec(p, t, ip);
3532
3533         /* Thread. */
3534         thread_ip_inc(p);
3535 }
3536
3537 static inline void
3538 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3539 {
3540         struct thread *t = &p->threads[p->thread_id];
3541         struct instruction *ip = t->ip;
3542
3543         /* Structs. */
3544         __instr_alu_shr_hi_exec(p, t, ip);
3545
3546         /* Thread. */
3547         thread_ip_inc(p);
3548 }
3549
3550 static inline void
3551 instr_alu_and_exec(struct rte_swx_pipeline *p)
3552 {
3553         struct thread *t = &p->threads[p->thread_id];
3554         struct instruction *ip = t->ip;
3555
3556         /* Structs. */
3557         __instr_alu_and_exec(p, t, ip);
3558
3559         /* Thread. */
3560         thread_ip_inc(p);
3561 }
3562
3563 static inline void
3564 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3565 {
3566         struct thread *t = &p->threads[p->thread_id];
3567         struct instruction *ip = t->ip;
3568
3569         /* Structs. */
3570         __instr_alu_and_mh_exec(p, t, ip);
3571
3572         /* Thread. */
3573         thread_ip_inc(p);
3574 }
3575
3576 static inline void
3577 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3578 {
3579         struct thread *t = &p->threads[p->thread_id];
3580         struct instruction *ip = t->ip;
3581
3582         /* Structs. */
3583         __instr_alu_and_hm_exec(p, t, ip);
3584
3585         /* Thread. */
3586         thread_ip_inc(p);
3587 }
3588
3589 static inline void
3590 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3591 {
3592         struct thread *t = &p->threads[p->thread_id];
3593         struct instruction *ip = t->ip;
3594
3595         /* Structs. */
3596         __instr_alu_and_hh_exec(p, t, ip);
3597
3598         /* Thread. */
3599         thread_ip_inc(p);
3600 }
3601
3602 static inline void
3603 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3604 {
3605         struct thread *t = &p->threads[p->thread_id];
3606         struct instruction *ip = t->ip;
3607
3608         /* Structs. */
3609         __instr_alu_and_i_exec(p, t, ip);
3610
3611         /* Thread. */
3612         thread_ip_inc(p);
3613 }
3614
3615 static inline void
3616 instr_alu_or_exec(struct rte_swx_pipeline *p)
3617 {
3618         struct thread *t = &p->threads[p->thread_id];
3619         struct instruction *ip = t->ip;
3620
3621         /* Structs. */
3622         __instr_alu_or_exec(p, t, ip);
3623
3624         /* Thread. */
3625         thread_ip_inc(p);
3626 }
3627
3628 static inline void
3629 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3630 {
3631         struct thread *t = &p->threads[p->thread_id];
3632         struct instruction *ip = t->ip;
3633
3634         /* Structs. */
3635         __instr_alu_or_mh_exec(p, t, ip);
3636
3637         /* Thread. */
3638         thread_ip_inc(p);
3639 }
3640
3641 static inline void
3642 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3643 {
3644         struct thread *t = &p->threads[p->thread_id];
3645         struct instruction *ip = t->ip;
3646
3647         /* Structs. */
3648         __instr_alu_or_hm_exec(p, t, ip);
3649
3650         /* Thread. */
3651         thread_ip_inc(p);
3652 }
3653
3654 static inline void
3655 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3656 {
3657         struct thread *t = &p->threads[p->thread_id];
3658         struct instruction *ip = t->ip;
3659
3660         /* Structs. */
3661         __instr_alu_or_hh_exec(p, t, ip);
3662
3663         /* Thread. */
3664         thread_ip_inc(p);
3665 }
3666
3667 static inline void
3668 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3669 {
3670         struct thread *t = &p->threads[p->thread_id];
3671         struct instruction *ip = t->ip;
3672
3673         /* Structs. */
3674         __instr_alu_or_i_exec(p, t, ip);
3675
3676         /* Thread. */
3677         thread_ip_inc(p);
3678 }
3679
3680 static inline void
3681 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3682 {
3683         struct thread *t = &p->threads[p->thread_id];
3684         struct instruction *ip = t->ip;
3685
3686         /* Structs. */
3687         __instr_alu_xor_exec(p, t, ip);
3688
3689         /* Thread. */
3690         thread_ip_inc(p);
3691 }
3692
3693 static inline void
3694 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3695 {
3696         struct thread *t = &p->threads[p->thread_id];
3697         struct instruction *ip = t->ip;
3698
3699         /* Structs. */
3700         __instr_alu_xor_mh_exec(p, t, ip);
3701
3702         /* Thread. */
3703         thread_ip_inc(p);
3704 }
3705
3706 static inline void
3707 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3708 {
3709         struct thread *t = &p->threads[p->thread_id];
3710         struct instruction *ip = t->ip;
3711
3712         /* Structs. */
3713         __instr_alu_xor_hm_exec(p, t, ip);
3714
3715         /* Thread. */
3716         thread_ip_inc(p);
3717 }
3718
3719 static inline void
3720 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3721 {
3722         struct thread *t = &p->threads[p->thread_id];
3723         struct instruction *ip = t->ip;
3724
3725         /* Structs. */
3726         __instr_alu_xor_hh_exec(p, t, ip);
3727
3728         /* Thread. */
3729         thread_ip_inc(p);
3730 }
3731
3732 static inline void
3733 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3734 {
3735         struct thread *t = &p->threads[p->thread_id];
3736         struct instruction *ip = t->ip;
3737
3738         /* Structs. */
3739         __instr_alu_xor_i_exec(p, t, ip);
3740
3741         /* Thread. */
3742         thread_ip_inc(p);
3743 }
3744
3745 static inline void
3746 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3747 {
3748         struct thread *t = &p->threads[p->thread_id];
3749         struct instruction *ip = t->ip;
3750
3751         /* Structs. */
3752         __instr_alu_ckadd_field_exec(p, t, ip);
3753
3754         /* Thread. */
3755         thread_ip_inc(p);
3756 }
3757
3758 static inline void
3759 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3760 {
3761         struct thread *t = &p->threads[p->thread_id];
3762         struct instruction *ip = t->ip;
3763
3764         /* Structs. */
3765         __instr_alu_cksub_field_exec(p, t, ip);
3766
3767         /* Thread. */
3768         thread_ip_inc(p);
3769 }
3770
3771 static inline void
3772 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3773 {
3774         struct thread *t = &p->threads[p->thread_id];
3775         struct instruction *ip = t->ip;
3776
3777         /* Structs. */
3778         __instr_alu_ckadd_struct20_exec(p, t, ip);
3779
3780         /* Thread. */
3781         thread_ip_inc(p);
3782 }
3783
3784 static inline void
3785 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3786 {
3787         struct thread *t = &p->threads[p->thread_id];
3788         struct instruction *ip = t->ip;
3789
3790         /* Structs. */
3791         __instr_alu_ckadd_struct_exec(p, t, ip);
3792
3793         /* Thread. */
3794         thread_ip_inc(p);
3795 }
3796
3797 /*
3798  * Register array.
3799  */
3800 static struct regarray *
3801 regarray_find(struct rte_swx_pipeline *p, const char *name);
3802
3803 static int
3804 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3805                       struct action *action,
3806                       char **tokens,
3807                       int n_tokens,
3808                       struct instruction *instr,
3809                       struct instruction_data *data __rte_unused)
3810 {
3811         char *regarray = tokens[1], *idx = tokens[2];
3812         struct regarray *r;
3813         struct field *fidx;
3814         uint32_t idx_struct_id, idx_val;
3815
3816         CHECK(n_tokens == 3, EINVAL);
3817
3818         r = regarray_find(p, regarray);
3819         CHECK(r, EINVAL);
3820
3821         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3822         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3823         if (fidx) {
3824                 CHECK(!fidx->var_size, EINVAL);
3825
3826                 instr->type = INSTR_REGPREFETCH_RM;
3827                 if (idx[0] == 'h')
3828                         instr->type = INSTR_REGPREFETCH_RH;
3829
3830                 instr->regarray.regarray_id = r->id;
3831                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3832                 instr->regarray.idx.n_bits = fidx->n_bits;
3833                 instr->regarray.idx.offset = fidx->offset / 8;
3834                 instr->regarray.dstsrc_val = 0; /* Unused. */
3835                 return 0;
3836         }
3837
3838         /* REGPREFETCH_RI. */
3839         idx_val = strtoul(idx, &idx, 0);
3840         CHECK(!idx[0], EINVAL);
3841
3842         instr->type = INSTR_REGPREFETCH_RI;
3843         instr->regarray.regarray_id = r->id;
3844         instr->regarray.idx_val = idx_val;
3845         instr->regarray.dstsrc_val = 0; /* Unused. */
3846         return 0;
3847 }
3848
3849 static int
3850 instr_regrd_translate(struct rte_swx_pipeline *p,
3851                       struct action *action,
3852                       char **tokens,
3853                       int n_tokens,
3854                       struct instruction *instr,
3855                       struct instruction_data *data __rte_unused)
3856 {
3857         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3858         struct regarray *r;
3859         struct field *fdst, *fidx;
3860         uint32_t dst_struct_id, idx_struct_id, idx_val;
3861
3862         CHECK(n_tokens == 4, EINVAL);
3863
3864         r = regarray_find(p, regarray);
3865         CHECK(r, EINVAL);
3866
3867         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3868         CHECK(fdst, EINVAL);
3869         CHECK(!fdst->var_size, EINVAL);
3870
3871         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3872         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3873         if (fidx) {
3874                 CHECK(!fidx->var_size, EINVAL);
3875
3876                 instr->type = INSTR_REGRD_MRM;
3877                 if (dst[0] == 'h' && idx[0] != 'h')
3878                         instr->type = INSTR_REGRD_HRM;
3879                 if (dst[0] != 'h' && idx[0] == 'h')
3880                         instr->type = INSTR_REGRD_MRH;
3881                 if (dst[0] == 'h' && idx[0] == 'h')
3882                         instr->type = INSTR_REGRD_HRH;
3883
3884                 instr->regarray.regarray_id = r->id;
3885                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3886                 instr->regarray.idx.n_bits = fidx->n_bits;
3887                 instr->regarray.idx.offset = fidx->offset / 8;
3888                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3889                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3890                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3891                 return 0;
3892         }
3893
3894         /* REGRD_MRI, REGRD_HRI. */
3895         idx_val = strtoul(idx, &idx, 0);
3896         CHECK(!idx[0], EINVAL);
3897
3898         instr->type = INSTR_REGRD_MRI;
3899         if (dst[0] == 'h')
3900                 instr->type = INSTR_REGRD_HRI;
3901
3902         instr->regarray.regarray_id = r->id;
3903         instr->regarray.idx_val = idx_val;
3904         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3905         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3906         instr->regarray.dstsrc.offset = fdst->offset / 8;
3907         return 0;
3908 }
3909
3910 static int
3911 instr_regwr_translate(struct rte_swx_pipeline *p,
3912                       struct action *action,
3913                       char **tokens,
3914                       int n_tokens,
3915                       struct instruction *instr,
3916                       struct instruction_data *data __rte_unused)
3917 {
3918         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3919         struct regarray *r;
3920         struct field *fidx, *fsrc;
3921         uint64_t src_val;
3922         uint32_t idx_struct_id, idx_val, src_struct_id;
3923
3924         CHECK(n_tokens == 4, EINVAL);
3925
3926         r = regarray_find(p, regarray);
3927         CHECK(r, EINVAL);
3928
3929         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3930         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3931         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3932         if (fidx && fsrc) {
3933                 CHECK(!fidx->var_size, EINVAL);
3934                 CHECK(!fsrc->var_size, EINVAL);
3935
3936                 instr->type = INSTR_REGWR_RMM;
3937                 if (idx[0] == 'h' && src[0] != 'h')
3938                         instr->type = INSTR_REGWR_RHM;
3939                 if (idx[0] != 'h' && src[0] == 'h')
3940                         instr->type = INSTR_REGWR_RMH;
3941                 if (idx[0] == 'h' && src[0] == 'h')
3942                         instr->type = INSTR_REGWR_RHH;
3943
3944                 instr->regarray.regarray_id = r->id;
3945                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3946                 instr->regarray.idx.n_bits = fidx->n_bits;
3947                 instr->regarray.idx.offset = fidx->offset / 8;
3948                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3949                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3950                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3951                 return 0;
3952         }
3953
3954         /* REGWR_RHI, REGWR_RMI. */
3955         if (fidx && !fsrc) {
3956                 CHECK(!fidx->var_size, EINVAL);
3957
3958                 src_val = strtoull(src, &src, 0);
3959                 CHECK(!src[0], EINVAL);
3960
3961                 instr->type = INSTR_REGWR_RMI;
3962                 if (idx[0] == 'h')
3963                         instr->type = INSTR_REGWR_RHI;
3964
3965                 instr->regarray.regarray_id = r->id;
3966                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3967                 instr->regarray.idx.n_bits = fidx->n_bits;
3968                 instr->regarray.idx.offset = fidx->offset / 8;
3969                 instr->regarray.dstsrc_val = src_val;
3970                 return 0;
3971         }
3972
3973         /* REGWR_RIH, REGWR_RIM. */
3974         if (!fidx && fsrc) {
3975                 idx_val = strtoul(idx, &idx, 0);
3976                 CHECK(!idx[0], EINVAL);
3977
3978                 CHECK(!fsrc->var_size, EINVAL);
3979
3980                 instr->type = INSTR_REGWR_RIM;
3981                 if (src[0] == 'h')
3982                         instr->type = INSTR_REGWR_RIH;
3983
3984                 instr->regarray.regarray_id = r->id;
3985                 instr->regarray.idx_val = idx_val;
3986                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3987                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3988                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3989                 return 0;
3990         }
3991
3992         /* REGWR_RII. */
3993         src_val = strtoull(src, &src, 0);
3994         CHECK(!src[0], EINVAL);
3995
3996         idx_val = strtoul(idx, &idx, 0);
3997         CHECK(!idx[0], EINVAL);
3998
3999         instr->type = INSTR_REGWR_RII;
4000         instr->regarray.idx_val = idx_val;
4001         instr->regarray.dstsrc_val = src_val;
4002
4003         return 0;
4004 }
4005
4006 static int
4007 instr_regadd_translate(struct rte_swx_pipeline *p,
4008                        struct action *action,
4009                        char **tokens,
4010                        int n_tokens,
4011                        struct instruction *instr,
4012                        struct instruction_data *data __rte_unused)
4013 {
4014         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4015         struct regarray *r;
4016         struct field *fidx, *fsrc;
4017         uint64_t src_val;
4018         uint32_t idx_struct_id, idx_val, src_struct_id;
4019
4020         CHECK(n_tokens == 4, EINVAL);
4021
4022         r = regarray_find(p, regarray);
4023         CHECK(r, EINVAL);
4024
4025         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4026         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4027         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4028         if (fidx && fsrc) {
4029                 CHECK(!fidx->var_size, EINVAL);
4030                 CHECK(!fsrc->var_size, EINVAL);
4031
4032                 instr->type = INSTR_REGADD_RMM;
4033                 if (idx[0] == 'h' && src[0] != 'h')
4034                         instr->type = INSTR_REGADD_RHM;
4035                 if (idx[0] != 'h' && src[0] == 'h')
4036                         instr->type = INSTR_REGADD_RMH;
4037                 if (idx[0] == 'h' && src[0] == 'h')
4038                         instr->type = INSTR_REGADD_RHH;
4039
4040                 instr->regarray.regarray_id = r->id;
4041                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4042                 instr->regarray.idx.n_bits = fidx->n_bits;
4043                 instr->regarray.idx.offset = fidx->offset / 8;
4044                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4045                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4046                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4047                 return 0;
4048         }
4049
4050         /* REGADD_RHI, REGADD_RMI. */
4051         if (fidx && !fsrc) {
4052                 CHECK(!fidx->var_size, EINVAL);
4053
4054                 src_val = strtoull(src, &src, 0);
4055                 CHECK(!src[0], EINVAL);
4056
4057                 instr->type = INSTR_REGADD_RMI;
4058                 if (idx[0] == 'h')
4059                         instr->type = INSTR_REGADD_RHI;
4060
4061                 instr->regarray.regarray_id = r->id;
4062                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4063                 instr->regarray.idx.n_bits = fidx->n_bits;
4064                 instr->regarray.idx.offset = fidx->offset / 8;
4065                 instr->regarray.dstsrc_val = src_val;
4066                 return 0;
4067         }
4068
4069         /* REGADD_RIH, REGADD_RIM. */
4070         if (!fidx && fsrc) {
4071                 idx_val = strtoul(idx, &idx, 0);
4072                 CHECK(!idx[0], EINVAL);
4073
4074                 CHECK(!fsrc->var_size, EINVAL);
4075
4076                 instr->type = INSTR_REGADD_RIM;
4077                 if (src[0] == 'h')
4078                         instr->type = INSTR_REGADD_RIH;
4079
4080                 instr->regarray.regarray_id = r->id;
4081                 instr->regarray.idx_val = idx_val;
4082                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4083                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4084                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4085                 return 0;
4086         }
4087
4088         /* REGADD_RII. */
4089         src_val = strtoull(src, &src, 0);
4090         CHECK(!src[0], EINVAL);
4091
4092         idx_val = strtoul(idx, &idx, 0);
4093         CHECK(!idx[0], EINVAL);
4094
4095         instr->type = INSTR_REGADD_RII;
4096         instr->regarray.idx_val = idx_val;
4097         instr->regarray.dstsrc_val = src_val;
4098         return 0;
4099 }
4100
4101 static inline void
4102 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4103 {
4104         struct thread *t = &p->threads[p->thread_id];
4105         struct instruction *ip = t->ip;
4106
4107         /* Structs. */
4108         __instr_regprefetch_rh_exec(p, t, ip);
4109
4110         /* Thread. */
4111         thread_ip_inc(p);
4112 }
4113
4114 static inline void
4115 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4116 {
4117         struct thread *t = &p->threads[p->thread_id];
4118         struct instruction *ip = t->ip;
4119
4120         /* Structs. */
4121         __instr_regprefetch_rm_exec(p, t, ip);
4122
4123         /* Thread. */
4124         thread_ip_inc(p);
4125 }
4126
4127 static inline void
4128 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4129 {
4130         struct thread *t = &p->threads[p->thread_id];
4131         struct instruction *ip = t->ip;
4132
4133         /* Structs. */
4134         __instr_regprefetch_ri_exec(p, t, ip);
4135
4136         /* Thread. */
4137         thread_ip_inc(p);
4138 }
4139
4140 static inline void
4141 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4142 {
4143         struct thread *t = &p->threads[p->thread_id];
4144         struct instruction *ip = t->ip;
4145
4146         /* Structs. */
4147         __instr_regrd_hrh_exec(p, t, ip);
4148
4149         /* Thread. */
4150         thread_ip_inc(p);
4151 }
4152
4153 static inline void
4154 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4155 {
4156         struct thread *t = &p->threads[p->thread_id];
4157         struct instruction *ip = t->ip;
4158
4159         /* Structs. */
4160         __instr_regrd_hrm_exec(p, t, ip);
4161
4162         /* Thread. */
4163         thread_ip_inc(p);
4164 }
4165
4166 static inline void
4167 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4168 {
4169         struct thread *t = &p->threads[p->thread_id];
4170         struct instruction *ip = t->ip;
4171
4172         /* Structs. */
4173         __instr_regrd_mrh_exec(p, t, ip);
4174
4175         /* Thread. */
4176         thread_ip_inc(p);
4177 }
4178
4179 static inline void
4180 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4181 {
4182         struct thread *t = &p->threads[p->thread_id];
4183         struct instruction *ip = t->ip;
4184
4185         /* Structs. */
4186         __instr_regrd_mrm_exec(p, t, ip);
4187
4188         /* Thread. */
4189         thread_ip_inc(p);
4190 }
4191
4192 static inline void
4193 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4194 {
4195         struct thread *t = &p->threads[p->thread_id];
4196         struct instruction *ip = t->ip;
4197
4198         /* Structs. */
4199         __instr_regrd_hri_exec(p, t, ip);
4200
4201         /* Thread. */
4202         thread_ip_inc(p);
4203 }
4204
4205 static inline void
4206 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4207 {
4208         struct thread *t = &p->threads[p->thread_id];
4209         struct instruction *ip = t->ip;
4210
4211         /* Structs. */
4212         __instr_regrd_mri_exec(p, t, ip);
4213
4214         /* Thread. */
4215         thread_ip_inc(p);
4216 }
4217
4218 static inline void
4219 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4220 {
4221         struct thread *t = &p->threads[p->thread_id];
4222         struct instruction *ip = t->ip;
4223
4224         /* Structs. */
4225         __instr_regwr_rhh_exec(p, t, ip);
4226
4227         /* Thread. */
4228         thread_ip_inc(p);
4229 }
4230
4231 static inline void
4232 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4233 {
4234         struct thread *t = &p->threads[p->thread_id];
4235         struct instruction *ip = t->ip;
4236
4237         /* Structs. */
4238         __instr_regwr_rhm_exec(p, t, ip);
4239
4240         /* Thread. */
4241         thread_ip_inc(p);
4242 }
4243
4244 static inline void
4245 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4246 {
4247         struct thread *t = &p->threads[p->thread_id];
4248         struct instruction *ip = t->ip;
4249
4250         /* Structs. */
4251         __instr_regwr_rmh_exec(p, t, ip);
4252
4253         /* Thread. */
4254         thread_ip_inc(p);
4255 }
4256
4257 static inline void
4258 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4259 {
4260         struct thread *t = &p->threads[p->thread_id];
4261         struct instruction *ip = t->ip;
4262
4263         /* Structs. */
4264         __instr_regwr_rmm_exec(p, t, ip);
4265
4266         /* Thread. */
4267         thread_ip_inc(p);
4268 }
4269
4270 static inline void
4271 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4272 {
4273         struct thread *t = &p->threads[p->thread_id];
4274         struct instruction *ip = t->ip;
4275
4276         /* Structs. */
4277         __instr_regwr_rhi_exec(p, t, ip);
4278
4279         /* Thread. */
4280         thread_ip_inc(p);
4281 }
4282
4283 static inline void
4284 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4285 {
4286         struct thread *t = &p->threads[p->thread_id];
4287         struct instruction *ip = t->ip;
4288
4289         /* Structs. */
4290         __instr_regwr_rmi_exec(p, t, ip);
4291
4292         /* Thread. */
4293         thread_ip_inc(p);
4294 }
4295
4296 static inline void
4297 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4298 {
4299         struct thread *t = &p->threads[p->thread_id];
4300         struct instruction *ip = t->ip;
4301
4302         /* Structs. */
4303         __instr_regwr_rih_exec(p, t, ip);
4304
4305         /* Thread. */
4306         thread_ip_inc(p);
4307 }
4308
4309 static inline void
4310 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4311 {
4312         struct thread *t = &p->threads[p->thread_id];
4313         struct instruction *ip = t->ip;
4314
4315         /* Structs. */
4316         __instr_regwr_rim_exec(p, t, ip);
4317
4318         /* Thread. */
4319         thread_ip_inc(p);
4320 }
4321
4322 static inline void
4323 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4324 {
4325         struct thread *t = &p->threads[p->thread_id];
4326         struct instruction *ip = t->ip;
4327
4328         /* Structs. */
4329         __instr_regwr_rii_exec(p, t, ip);
4330
4331         /* Thread. */
4332         thread_ip_inc(p);
4333 }
4334
4335 static inline void
4336 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4337 {
4338         struct thread *t = &p->threads[p->thread_id];
4339         struct instruction *ip = t->ip;
4340
4341         /* Structs. */
4342         __instr_regadd_rhh_exec(p, t, ip);
4343
4344         /* Thread. */
4345         thread_ip_inc(p);
4346 }
4347
4348 static inline void
4349 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4350 {
4351         struct thread *t = &p->threads[p->thread_id];
4352         struct instruction *ip = t->ip;
4353
4354         /* Structs. */
4355         __instr_regadd_rhm_exec(p, t, ip);
4356
4357         /* Thread. */
4358         thread_ip_inc(p);
4359 }
4360
4361 static inline void
4362 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4363 {
4364         struct thread *t = &p->threads[p->thread_id];
4365         struct instruction *ip = t->ip;
4366
4367         /* Structs. */
4368         __instr_regadd_rmh_exec(p, t, ip);
4369
4370         /* Thread. */
4371         thread_ip_inc(p);
4372 }
4373
4374 static inline void
4375 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4376 {
4377         struct thread *t = &p->threads[p->thread_id];
4378         struct instruction *ip = t->ip;
4379
4380         /* Structs. */
4381         __instr_regadd_rmm_exec(p, t, ip);
4382
4383         /* Thread. */
4384         thread_ip_inc(p);
4385 }
4386
4387 static inline void
4388 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4389 {
4390         struct thread *t = &p->threads[p->thread_id];
4391         struct instruction *ip = t->ip;
4392
4393         /* Structs. */
4394         __instr_regadd_rhi_exec(p, t, ip);
4395
4396         /* Thread. */
4397         thread_ip_inc(p);
4398 }
4399
4400 static inline void
4401 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4402 {
4403         struct thread *t = &p->threads[p->thread_id];
4404         struct instruction *ip = t->ip;
4405
4406         /* Structs. */
4407         __instr_regadd_rmi_exec(p, t, ip);
4408
4409         /* Thread. */
4410         thread_ip_inc(p);
4411 }
4412
4413 static inline void
4414 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4415 {
4416         struct thread *t = &p->threads[p->thread_id];
4417         struct instruction *ip = t->ip;
4418
4419         /* Structs. */
4420         __instr_regadd_rih_exec(p, t, ip);
4421
4422         /* Thread. */
4423         thread_ip_inc(p);
4424 }
4425
4426 static inline void
4427 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4428 {
4429         struct thread *t = &p->threads[p->thread_id];
4430         struct instruction *ip = t->ip;
4431
4432         /* Structs. */
4433         __instr_regadd_rim_exec(p, t, ip);
4434
4435         /* Thread. */
4436         thread_ip_inc(p);
4437 }
4438
4439 static inline void
4440 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4441 {
4442         struct thread *t = &p->threads[p->thread_id];
4443         struct instruction *ip = t->ip;
4444
4445         /* Structs. */
4446         __instr_regadd_rii_exec(p, t, ip);
4447
4448         /* Thread. */
4449         thread_ip_inc(p);
4450 }
4451
4452 /*
4453  * metarray.
4454  */
4455 static struct metarray *
4456 metarray_find(struct rte_swx_pipeline *p, const char *name);
4457
4458 static int
4459 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4460                             struct action *action,
4461                             char **tokens,
4462                             int n_tokens,
4463                             struct instruction *instr,
4464                             struct instruction_data *data __rte_unused)
4465 {
4466         char *metarray = tokens[1], *idx = tokens[2];
4467         struct metarray *m;
4468         struct field *fidx;
4469         uint32_t idx_struct_id, idx_val;
4470
4471         CHECK(n_tokens == 3, EINVAL);
4472
4473         m = metarray_find(p, metarray);
4474         CHECK(m, EINVAL);
4475
4476         /* METPREFETCH_H, METPREFETCH_M. */
4477         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4478         if (fidx) {
4479                 CHECK(!fidx->var_size, EINVAL);
4480
4481                 instr->type = INSTR_METPREFETCH_M;
4482                 if (idx[0] == 'h')
4483                         instr->type = INSTR_METPREFETCH_H;
4484
4485                 instr->meter.metarray_id = m->id;
4486                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4487                 instr->meter.idx.n_bits = fidx->n_bits;
4488                 instr->meter.idx.offset = fidx->offset / 8;
4489                 return 0;
4490         }
4491
4492         /* METPREFETCH_I. */
4493         idx_val = strtoul(idx, &idx, 0);
4494         CHECK(!idx[0], EINVAL);
4495
4496         instr->type = INSTR_METPREFETCH_I;
4497         instr->meter.metarray_id = m->id;
4498         instr->meter.idx_val = idx_val;
4499         return 0;
4500 }
4501
4502 static int
4503 instr_meter_translate(struct rte_swx_pipeline *p,
4504                       struct action *action,
4505                       char **tokens,
4506                       int n_tokens,
4507                       struct instruction *instr,
4508                       struct instruction_data *data __rte_unused)
4509 {
4510         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4511         char *color_in = tokens[4], *color_out = tokens[5];
4512         struct metarray *m;
4513         struct field *fidx, *flength, *fcin, *fcout;
4514         uint32_t idx_struct_id, length_struct_id;
4515         uint32_t color_in_struct_id, color_out_struct_id;
4516
4517         CHECK(n_tokens == 6, EINVAL);
4518
4519         m = metarray_find(p, metarray);
4520         CHECK(m, EINVAL);
4521
4522         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4523
4524         flength = struct_field_parse(p, action, length, &length_struct_id);
4525         CHECK(flength, EINVAL);
4526         CHECK(!flength->var_size, EINVAL);
4527
4528         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4529
4530         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4531         CHECK(fcout, EINVAL);
4532         CHECK(!fcout->var_size, EINVAL);
4533
4534         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4535         if (fidx && fcin) {
4536                 CHECK(!fidx->var_size, EINVAL);
4537                 CHECK(!fcin->var_size, EINVAL);
4538
4539                 instr->type = INSTR_METER_MMM;
4540                 if (idx[0] == 'h' && length[0] == 'h')
4541                         instr->type = INSTR_METER_HHM;
4542                 if (idx[0] == 'h' && length[0] != 'h')
4543                         instr->type = INSTR_METER_HMM;
4544                 if (idx[0] != 'h' && length[0] == 'h')
4545                         instr->type = INSTR_METER_MHM;
4546
4547                 instr->meter.metarray_id = m->id;
4548
4549                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4550                 instr->meter.idx.n_bits = fidx->n_bits;
4551                 instr->meter.idx.offset = fidx->offset / 8;
4552
4553                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4554                 instr->meter.length.n_bits = flength->n_bits;
4555                 instr->meter.length.offset = flength->offset / 8;
4556
4557                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4558                 instr->meter.color_in.n_bits = fcin->n_bits;
4559                 instr->meter.color_in.offset = fcin->offset / 8;
4560
4561                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4562                 instr->meter.color_out.n_bits = fcout->n_bits;
4563                 instr->meter.color_out.offset = fcout->offset / 8;
4564         }
4565
4566         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4567         if (fidx && !fcin) {
4568                 uint32_t color_in_val;
4569
4570                 CHECK(!fidx->var_size, EINVAL);
4571
4572                 color_in_val = strtoul(color_in, &color_in, 0);
4573                 CHECK(!color_in[0], EINVAL);
4574
4575                 instr->type = INSTR_METER_MMI;
4576                 if (idx[0] == 'h' && length[0] == 'h')
4577                         instr->type = INSTR_METER_HHI;
4578                 if (idx[0] == 'h' && length[0] != 'h')
4579                         instr->type = INSTR_METER_HMI;
4580                 if (idx[0] != 'h' && length[0] == 'h')
4581                         instr->type = INSTR_METER_MHI;
4582
4583                 instr->meter.metarray_id = m->id;
4584
4585                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4586                 instr->meter.idx.n_bits = fidx->n_bits;
4587                 instr->meter.idx.offset = fidx->offset / 8;
4588
4589                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4590                 instr->meter.length.n_bits = flength->n_bits;
4591                 instr->meter.length.offset = flength->offset / 8;
4592
4593                 instr->meter.color_in_val = color_in_val;
4594
4595                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4596                 instr->meter.color_out.n_bits = fcout->n_bits;
4597                 instr->meter.color_out.offset = fcout->offset / 8;
4598         }
4599
4600         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4601         if (!fidx && fcin) {
4602                 uint32_t idx_val;
4603
4604                 idx_val = strtoul(idx, &idx, 0);
4605                 CHECK(!idx[0], EINVAL);
4606
4607                 CHECK(!fcin->var_size, EINVAL);
4608
4609                 instr->type = INSTR_METER_IMM;
4610                 if (length[0] == 'h')
4611                         instr->type = INSTR_METER_IHM;
4612
4613                 instr->meter.metarray_id = m->id;
4614
4615                 instr->meter.idx_val = idx_val;
4616
4617                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4618                 instr->meter.length.n_bits = flength->n_bits;
4619                 instr->meter.length.offset = flength->offset / 8;
4620
4621                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4622                 instr->meter.color_in.n_bits = fcin->n_bits;
4623                 instr->meter.color_in.offset = fcin->offset / 8;
4624
4625                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4626                 instr->meter.color_out.n_bits = fcout->n_bits;
4627                 instr->meter.color_out.offset = fcout->offset / 8;
4628         }
4629
4630         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4631         if (!fidx && !fcin) {
4632                 uint32_t idx_val, color_in_val;
4633
4634                 idx_val = strtoul(idx, &idx, 0);
4635                 CHECK(!idx[0], EINVAL);
4636
4637                 color_in_val = strtoul(color_in, &color_in, 0);
4638                 CHECK(!color_in[0], EINVAL);
4639
4640                 instr->type = INSTR_METER_IMI;
4641                 if (length[0] == 'h')
4642                         instr->type = INSTR_METER_IHI;
4643
4644                 instr->meter.metarray_id = m->id;
4645
4646                 instr->meter.idx_val = idx_val;
4647
4648                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4649                 instr->meter.length.n_bits = flength->n_bits;
4650                 instr->meter.length.offset = flength->offset / 8;
4651
4652                 instr->meter.color_in_val = color_in_val;
4653
4654                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4655                 instr->meter.color_out.n_bits = fcout->n_bits;
4656                 instr->meter.color_out.offset = fcout->offset / 8;
4657         }
4658
4659         return 0;
4660 }
4661
4662 static inline void
4663 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4664 {
4665         struct thread *t = &p->threads[p->thread_id];
4666         struct instruction *ip = t->ip;
4667
4668         /* Structs. */
4669         __instr_metprefetch_h_exec(p, t, ip);
4670
4671         /* Thread. */
4672         thread_ip_inc(p);
4673 }
4674
4675 static inline void
4676 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4677 {
4678         struct thread *t = &p->threads[p->thread_id];
4679         struct instruction *ip = t->ip;
4680
4681         /* Structs. */
4682         __instr_metprefetch_m_exec(p, t, ip);
4683
4684         /* Thread. */
4685         thread_ip_inc(p);
4686 }
4687
4688 static inline void
4689 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4690 {
4691         struct thread *t = &p->threads[p->thread_id];
4692         struct instruction *ip = t->ip;
4693
4694         /* Structs. */
4695         __instr_metprefetch_i_exec(p, t, ip);
4696
4697         /* Thread. */
4698         thread_ip_inc(p);
4699 }
4700
4701 static inline void
4702 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4703 {
4704         struct thread *t = &p->threads[p->thread_id];
4705         struct instruction *ip = t->ip;
4706
4707         /* Structs. */
4708         __instr_meter_hhm_exec(p, t, ip);
4709
4710         /* Thread. */
4711         thread_ip_inc(p);
4712 }
4713
4714 static inline void
4715 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4716 {
4717         struct thread *t = &p->threads[p->thread_id];
4718         struct instruction *ip = t->ip;
4719
4720         /* Structs. */
4721         __instr_meter_hhi_exec(p, t, ip);
4722
4723         /* Thread. */
4724         thread_ip_inc(p);
4725 }
4726
4727 static inline void
4728 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4729 {
4730         struct thread *t = &p->threads[p->thread_id];
4731         struct instruction *ip = t->ip;
4732
4733         /* Structs. */
4734         __instr_meter_hmm_exec(p, t, ip);
4735
4736         /* Thread. */
4737         thread_ip_inc(p);
4738 }
4739
4740 static inline void
4741 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4742 {
4743         struct thread *t = &p->threads[p->thread_id];
4744         struct instruction *ip = t->ip;
4745
4746         /* Structs. */
4747         __instr_meter_hmi_exec(p, t, ip);
4748
4749         /* Thread. */
4750         thread_ip_inc(p);
4751 }
4752
4753 static inline void
4754 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4755 {
4756         struct thread *t = &p->threads[p->thread_id];
4757         struct instruction *ip = t->ip;
4758
4759         /* Structs. */
4760         __instr_meter_mhm_exec(p, t, ip);
4761
4762         /* Thread. */
4763         thread_ip_inc(p);
4764 }
4765
4766 static inline void
4767 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4768 {
4769         struct thread *t = &p->threads[p->thread_id];
4770         struct instruction *ip = t->ip;
4771
4772         /* Structs. */
4773         __instr_meter_mhi_exec(p, t, ip);
4774
4775         /* Thread. */
4776         thread_ip_inc(p);
4777 }
4778
4779 static inline void
4780 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4781 {
4782         struct thread *t = &p->threads[p->thread_id];
4783         struct instruction *ip = t->ip;
4784
4785         /* Structs. */
4786         __instr_meter_mmm_exec(p, t, ip);
4787
4788         /* Thread. */
4789         thread_ip_inc(p);
4790 }
4791
4792 static inline void
4793 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4794 {
4795         struct thread *t = &p->threads[p->thread_id];
4796         struct instruction *ip = t->ip;
4797
4798         /* Structs. */
4799         __instr_meter_mmi_exec(p, t, ip);
4800
4801         /* Thread. */
4802         thread_ip_inc(p);
4803 }
4804
4805 static inline void
4806 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4807 {
4808         struct thread *t = &p->threads[p->thread_id];
4809         struct instruction *ip = t->ip;
4810
4811         /* Structs. */
4812         __instr_meter_ihm_exec(p, t, ip);
4813
4814         /* Thread. */
4815         thread_ip_inc(p);
4816 }
4817
4818 static inline void
4819 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4820 {
4821         struct thread *t = &p->threads[p->thread_id];
4822         struct instruction *ip = t->ip;
4823
4824         /* Structs. */
4825         __instr_meter_ihi_exec(p, t, ip);
4826
4827         /* Thread. */
4828         thread_ip_inc(p);
4829 }
4830
4831 static inline void
4832 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4833 {
4834         struct thread *t = &p->threads[p->thread_id];
4835         struct instruction *ip = t->ip;
4836
4837         /* Structs. */
4838         __instr_meter_imm_exec(p, t, ip);
4839
4840         /* Thread. */
4841         thread_ip_inc(p);
4842 }
4843
4844 static inline void
4845 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4846 {
4847         struct thread *t = &p->threads[p->thread_id];
4848         struct instruction *ip = t->ip;
4849
4850         /* Structs. */
4851         __instr_meter_imi_exec(p, t, ip);
4852
4853         /* Thread. */
4854         thread_ip_inc(p);
4855 }
4856
4857 /*
4858  * jmp.
4859  */
4860 static int
4861 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4862                     struct action *action __rte_unused,
4863                     char **tokens,
4864                     int n_tokens,
4865                     struct instruction *instr,
4866                     struct instruction_data *data)
4867 {
4868         CHECK(n_tokens == 2, EINVAL);
4869
4870         strcpy(data->jmp_label, tokens[1]);
4871
4872         instr->type = INSTR_JMP;
4873         instr->jmp.ip = NULL; /* Resolved later. */
4874         return 0;
4875 }
4876
4877 static int
4878 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4879                           struct action *action __rte_unused,
4880                           char **tokens,
4881                           int n_tokens,
4882                           struct instruction *instr,
4883                           struct instruction_data *data)
4884 {
4885         struct header *h;
4886
4887         CHECK(n_tokens == 3, EINVAL);
4888
4889         strcpy(data->jmp_label, tokens[1]);
4890
4891         h = header_parse(p, tokens[2]);
4892         CHECK(h, EINVAL);
4893
4894         instr->type = INSTR_JMP_VALID;
4895         instr->jmp.ip = NULL; /* Resolved later. */
4896         instr->jmp.header_id = h->id;
4897         return 0;
4898 }
4899
4900 static int
4901 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4902                             struct action *action __rte_unused,
4903                             char **tokens,
4904                             int n_tokens,
4905                             struct instruction *instr,
4906                             struct instruction_data *data)
4907 {
4908         struct header *h;
4909
4910         CHECK(n_tokens == 3, EINVAL);
4911
4912         strcpy(data->jmp_label, tokens[1]);
4913
4914         h = header_parse(p, tokens[2]);
4915         CHECK(h, EINVAL);
4916
4917         instr->type = INSTR_JMP_INVALID;
4918         instr->jmp.ip = NULL; /* Resolved later. */
4919         instr->jmp.header_id = h->id;
4920         return 0;
4921 }
4922
4923 static int
4924 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4925                         struct action *action,
4926                         char **tokens,
4927                         int n_tokens,
4928                         struct instruction *instr,
4929                         struct instruction_data *data)
4930 {
4931         CHECK(!action, EINVAL);
4932         CHECK(n_tokens == 2, EINVAL);
4933
4934         strcpy(data->jmp_label, tokens[1]);
4935
4936         instr->type = INSTR_JMP_HIT;
4937         instr->jmp.ip = NULL; /* Resolved later. */
4938         return 0;
4939 }
4940
4941 static int
4942 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4943                          struct action *action,
4944                          char **tokens,
4945                          int n_tokens,
4946                          struct instruction *instr,
4947                          struct instruction_data *data)
4948 {
4949         CHECK(!action, EINVAL);
4950         CHECK(n_tokens == 2, EINVAL);
4951
4952         strcpy(data->jmp_label, tokens[1]);
4953
4954         instr->type = INSTR_JMP_MISS;
4955         instr->jmp.ip = NULL; /* Resolved later. */
4956         return 0;
4957 }
4958
4959 static int
4960 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4961                                struct action *action,
4962                                char **tokens,
4963                                int n_tokens,
4964                                struct instruction *instr,
4965                                struct instruction_data *data)
4966 {
4967         struct action *a;
4968
4969         CHECK(!action, EINVAL);
4970         CHECK(n_tokens == 3, EINVAL);
4971
4972         strcpy(data->jmp_label, tokens[1]);
4973
4974         a = action_find(p, tokens[2]);
4975         CHECK(a, EINVAL);
4976
4977         instr->type = INSTR_JMP_ACTION_HIT;
4978         instr->jmp.ip = NULL; /* Resolved later. */
4979         instr->jmp.action_id = a->id;
4980         return 0;
4981 }
4982
4983 static int
4984 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4985                                 struct action *action,
4986                                 char **tokens,
4987                                 int n_tokens,
4988                                 struct instruction *instr,
4989                                 struct instruction_data *data)
4990 {
4991         struct action *a;
4992
4993         CHECK(!action, EINVAL);
4994         CHECK(n_tokens == 3, EINVAL);
4995
4996         strcpy(data->jmp_label, tokens[1]);
4997
4998         a = action_find(p, tokens[2]);
4999         CHECK(a, EINVAL);
5000
5001         instr->type = INSTR_JMP_ACTION_MISS;
5002         instr->jmp.ip = NULL; /* Resolved later. */
5003         instr->jmp.action_id = a->id;
5004         return 0;
5005 }
5006
5007 static int
5008 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5009                        struct action *action,
5010                        char **tokens,
5011                        int n_tokens,
5012                        struct instruction *instr,
5013                        struct instruction_data *data)
5014 {
5015         char *a = tokens[2], *b = tokens[3];
5016         struct field *fa, *fb;
5017         uint64_t b_val;
5018         uint32_t a_struct_id, b_struct_id;
5019
5020         CHECK(n_tokens == 4, EINVAL);
5021
5022         strcpy(data->jmp_label, tokens[1]);
5023
5024         fa = struct_field_parse(p, action, a, &a_struct_id);
5025         CHECK(fa, EINVAL);
5026         CHECK(!fa->var_size, EINVAL);
5027
5028         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5029         fb = struct_field_parse(p, action, b, &b_struct_id);
5030         if (fb) {
5031                 CHECK(!fb->var_size, EINVAL);
5032
5033                 instr->type = INSTR_JMP_EQ;
5034                 if (a[0] != 'h' && b[0] == 'h')
5035                         instr->type = INSTR_JMP_EQ_MH;
5036                 if (a[0] == 'h' && b[0] != 'h')
5037                         instr->type = INSTR_JMP_EQ_HM;
5038                 if (a[0] == 'h' && b[0] == 'h')
5039                         instr->type = INSTR_JMP_EQ_HH;
5040                 instr->jmp.ip = NULL; /* Resolved later. */
5041
5042                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5043                 instr->jmp.a.n_bits = fa->n_bits;
5044                 instr->jmp.a.offset = fa->offset / 8;
5045                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5046                 instr->jmp.b.n_bits = fb->n_bits;
5047                 instr->jmp.b.offset = fb->offset / 8;
5048                 return 0;
5049         }
5050
5051         /* JMP_EQ_I. */
5052         b_val = strtoull(b, &b, 0);
5053         CHECK(!b[0], EINVAL);
5054
5055         if (a[0] == 'h')
5056                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5057
5058         instr->type = INSTR_JMP_EQ_I;
5059         instr->jmp.ip = NULL; /* Resolved later. */
5060         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5061         instr->jmp.a.n_bits = fa->n_bits;
5062         instr->jmp.a.offset = fa->offset / 8;
5063         instr->jmp.b_val = b_val;
5064         return 0;
5065 }
5066
5067 static int
5068 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5069                         struct action *action,
5070                         char **tokens,
5071                         int n_tokens,
5072                         struct instruction *instr,
5073                         struct instruction_data *data)
5074 {
5075         char *a = tokens[2], *b = tokens[3];
5076         struct field *fa, *fb;
5077         uint64_t b_val;
5078         uint32_t a_struct_id, b_struct_id;
5079
5080         CHECK(n_tokens == 4, EINVAL);
5081
5082         strcpy(data->jmp_label, tokens[1]);
5083
5084         fa = struct_field_parse(p, action, a, &a_struct_id);
5085         CHECK(fa, EINVAL);
5086         CHECK(!fa->var_size, EINVAL);
5087
5088         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5089         fb = struct_field_parse(p, action, b, &b_struct_id);
5090         if (fb) {
5091                 CHECK(!fb->var_size, EINVAL);
5092
5093                 instr->type = INSTR_JMP_NEQ;
5094                 if (a[0] != 'h' && b[0] == 'h')
5095                         instr->type = INSTR_JMP_NEQ_MH;
5096                 if (a[0] == 'h' && b[0] != 'h')
5097                         instr->type = INSTR_JMP_NEQ_HM;
5098                 if (a[0] == 'h' && b[0] == 'h')
5099                         instr->type = INSTR_JMP_NEQ_HH;
5100                 instr->jmp.ip = NULL; /* Resolved later. */
5101
5102                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5103                 instr->jmp.a.n_bits = fa->n_bits;
5104                 instr->jmp.a.offset = fa->offset / 8;
5105                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5106                 instr->jmp.b.n_bits = fb->n_bits;
5107                 instr->jmp.b.offset = fb->offset / 8;
5108                 return 0;
5109         }
5110
5111         /* JMP_NEQ_I. */
5112         b_val = strtoull(b, &b, 0);
5113         CHECK(!b[0], EINVAL);
5114
5115         if (a[0] == 'h')
5116                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5117
5118         instr->type = INSTR_JMP_NEQ_I;
5119         instr->jmp.ip = NULL; /* Resolved later. */
5120         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5121         instr->jmp.a.n_bits = fa->n_bits;
5122         instr->jmp.a.offset = fa->offset / 8;
5123         instr->jmp.b_val = b_val;
5124         return 0;
5125 }
5126
5127 static int
5128 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5129                        struct action *action,
5130                        char **tokens,
5131                        int n_tokens,
5132                        struct instruction *instr,
5133                        struct instruction_data *data)
5134 {
5135         char *a = tokens[2], *b = tokens[3];
5136         struct field *fa, *fb;
5137         uint64_t b_val;
5138         uint32_t a_struct_id, b_struct_id;
5139
5140         CHECK(n_tokens == 4, EINVAL);
5141
5142         strcpy(data->jmp_label, tokens[1]);
5143
5144         fa = struct_field_parse(p, action, a, &a_struct_id);
5145         CHECK(fa, EINVAL);
5146         CHECK(!fa->var_size, EINVAL);
5147
5148         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5149         fb = struct_field_parse(p, action, b, &b_struct_id);
5150         if (fb) {
5151                 CHECK(!fb->var_size, EINVAL);
5152
5153                 instr->type = INSTR_JMP_LT;
5154                 if (a[0] == 'h' && b[0] != 'h')
5155                         instr->type = INSTR_JMP_LT_HM;
5156                 if (a[0] != 'h' && b[0] == 'h')
5157                         instr->type = INSTR_JMP_LT_MH;
5158                 if (a[0] == 'h' && b[0] == 'h')
5159                         instr->type = INSTR_JMP_LT_HH;
5160                 instr->jmp.ip = NULL; /* Resolved later. */
5161
5162                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5163                 instr->jmp.a.n_bits = fa->n_bits;
5164                 instr->jmp.a.offset = fa->offset / 8;
5165                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5166                 instr->jmp.b.n_bits = fb->n_bits;
5167                 instr->jmp.b.offset = fb->offset / 8;
5168                 return 0;
5169         }
5170
5171         /* JMP_LT_MI, JMP_LT_HI. */
5172         b_val = strtoull(b, &b, 0);
5173         CHECK(!b[0], EINVAL);
5174
5175         instr->type = INSTR_JMP_LT_MI;
5176         if (a[0] == 'h')
5177                 instr->type = INSTR_JMP_LT_HI;
5178         instr->jmp.ip = NULL; /* Resolved later. */
5179
5180         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5181         instr->jmp.a.n_bits = fa->n_bits;
5182         instr->jmp.a.offset = fa->offset / 8;
5183         instr->jmp.b_val = b_val;
5184         return 0;
5185 }
5186
5187 static int
5188 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5189                        struct action *action,
5190                        char **tokens,
5191                        int n_tokens,
5192                        struct instruction *instr,
5193                        struct instruction_data *data)
5194 {
5195         char *a = tokens[2], *b = tokens[3];
5196         struct field *fa, *fb;
5197         uint64_t b_val;
5198         uint32_t a_struct_id, b_struct_id;
5199
5200         CHECK(n_tokens == 4, EINVAL);
5201
5202         strcpy(data->jmp_label, tokens[1]);
5203
5204         fa = struct_field_parse(p, action, a, &a_struct_id);
5205         CHECK(fa, EINVAL);
5206         CHECK(!fa->var_size, EINVAL);
5207
5208         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5209         fb = struct_field_parse(p, action, b, &b_struct_id);
5210         if (fb) {
5211                 CHECK(!fb->var_size, EINVAL);
5212
5213                 instr->type = INSTR_JMP_GT;
5214                 if (a[0] == 'h' && b[0] != 'h')
5215                         instr->type = INSTR_JMP_GT_HM;
5216                 if (a[0] != 'h' && b[0] == 'h')
5217                         instr->type = INSTR_JMP_GT_MH;
5218                 if (a[0] == 'h' && b[0] == 'h')
5219                         instr->type = INSTR_JMP_GT_HH;
5220                 instr->jmp.ip = NULL; /* Resolved later. */
5221
5222                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5223                 instr->jmp.a.n_bits = fa->n_bits;
5224                 instr->jmp.a.offset = fa->offset / 8;
5225                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5226                 instr->jmp.b.n_bits = fb->n_bits;
5227                 instr->jmp.b.offset = fb->offset / 8;
5228                 return 0;
5229         }
5230
5231         /* JMP_GT_MI, JMP_GT_HI. */
5232         b_val = strtoull(b, &b, 0);
5233         CHECK(!b[0], EINVAL);
5234
5235         instr->type = INSTR_JMP_GT_MI;
5236         if (a[0] == 'h')
5237                 instr->type = INSTR_JMP_GT_HI;
5238         instr->jmp.ip = NULL; /* Resolved later. */
5239
5240         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5241         instr->jmp.a.n_bits = fa->n_bits;
5242         instr->jmp.a.offset = fa->offset / 8;
5243         instr->jmp.b_val = b_val;
5244         return 0;
5245 }
5246
5247 static inline void
5248 instr_jmp_exec(struct rte_swx_pipeline *p)
5249 {
5250         struct thread *t = &p->threads[p->thread_id];
5251         struct instruction *ip = t->ip;
5252
5253         TRACE("[Thread %2u] jmp\n", p->thread_id);
5254
5255         thread_ip_set(t, ip->jmp.ip);
5256 }
5257
5258 static inline void
5259 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5260 {
5261         struct thread *t = &p->threads[p->thread_id];
5262         struct instruction *ip = t->ip;
5263         uint32_t header_id = ip->jmp.header_id;
5264
5265         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5266
5267         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5268 }
5269
5270 static inline void
5271 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5272 {
5273         struct thread *t = &p->threads[p->thread_id];
5274         struct instruction *ip = t->ip;
5275         uint32_t header_id = ip->jmp.header_id;
5276
5277         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5278
5279         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5280 }
5281
5282 static inline void
5283 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5284 {
5285         struct thread *t = &p->threads[p->thread_id];
5286         struct instruction *ip = t->ip;
5287         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5288
5289         TRACE("[Thread %2u] jmph\n", p->thread_id);
5290
5291         t->ip = ip_next[t->hit];
5292 }
5293
5294 static inline void
5295 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5296 {
5297         struct thread *t = &p->threads[p->thread_id];
5298         struct instruction *ip = t->ip;
5299         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5300
5301         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5302
5303         t->ip = ip_next[t->hit];
5304 }
5305
5306 static inline void
5307 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5308 {
5309         struct thread *t = &p->threads[p->thread_id];
5310         struct instruction *ip = t->ip;
5311
5312         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5313
5314         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5315 }
5316
5317 static inline void
5318 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5319 {
5320         struct thread *t = &p->threads[p->thread_id];
5321         struct instruction *ip = t->ip;
5322
5323         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5324
5325         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5326 }
5327
5328 static inline void
5329 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5330 {
5331         struct thread *t = &p->threads[p->thread_id];
5332         struct instruction *ip = t->ip;
5333
5334         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5335
5336         JMP_CMP(t, ip, ==);
5337 }
5338
5339 static inline void
5340 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5341 {
5342         struct thread *t = &p->threads[p->thread_id];
5343         struct instruction *ip = t->ip;
5344
5345         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5346
5347         JMP_CMP_MH(t, ip, ==);
5348 }
5349
5350 static inline void
5351 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5352 {
5353         struct thread *t = &p->threads[p->thread_id];
5354         struct instruction *ip = t->ip;
5355
5356         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5357
5358         JMP_CMP_HM(t, ip, ==);
5359 }
5360
5361 static inline void
5362 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5363 {
5364         struct thread *t = &p->threads[p->thread_id];
5365         struct instruction *ip = t->ip;
5366
5367         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5368
5369         JMP_CMP_HH_FAST(t, ip, ==);
5370 }
5371
5372 static inline void
5373 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5374 {
5375         struct thread *t = &p->threads[p->thread_id];
5376         struct instruction *ip = t->ip;
5377
5378         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5379
5380         JMP_CMP_I(t, ip, ==);
5381 }
5382
5383 static inline void
5384 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5385 {
5386         struct thread *t = &p->threads[p->thread_id];
5387         struct instruction *ip = t->ip;
5388
5389         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5390
5391         JMP_CMP(t, ip, !=);
5392 }
5393
5394 static inline void
5395 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5396 {
5397         struct thread *t = &p->threads[p->thread_id];
5398         struct instruction *ip = t->ip;
5399
5400         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5401
5402         JMP_CMP_MH(t, ip, !=);
5403 }
5404
5405 static inline void
5406 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5407 {
5408         struct thread *t = &p->threads[p->thread_id];
5409         struct instruction *ip = t->ip;
5410
5411         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5412
5413         JMP_CMP_HM(t, ip, !=);
5414 }
5415
5416 static inline void
5417 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5418 {
5419         struct thread *t = &p->threads[p->thread_id];
5420         struct instruction *ip = t->ip;
5421
5422         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5423
5424         JMP_CMP_HH_FAST(t, ip, !=);
5425 }
5426
5427 static inline void
5428 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5429 {
5430         struct thread *t = &p->threads[p->thread_id];
5431         struct instruction *ip = t->ip;
5432
5433         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5434
5435         JMP_CMP_I(t, ip, !=);
5436 }
5437
5438 static inline void
5439 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5440 {
5441         struct thread *t = &p->threads[p->thread_id];
5442         struct instruction *ip = t->ip;
5443
5444         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5445
5446         JMP_CMP(t, ip, <);
5447 }
5448
5449 static inline void
5450 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5451 {
5452         struct thread *t = &p->threads[p->thread_id];
5453         struct instruction *ip = t->ip;
5454
5455         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5456
5457         JMP_CMP_MH(t, ip, <);
5458 }
5459
5460 static inline void
5461 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5462 {
5463         struct thread *t = &p->threads[p->thread_id];
5464         struct instruction *ip = t->ip;
5465
5466         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5467
5468         JMP_CMP_HM(t, ip, <);
5469 }
5470
5471 static inline void
5472 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5473 {
5474         struct thread *t = &p->threads[p->thread_id];
5475         struct instruction *ip = t->ip;
5476
5477         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5478
5479         JMP_CMP_HH(t, ip, <);
5480 }
5481
5482 static inline void
5483 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5484 {
5485         struct thread *t = &p->threads[p->thread_id];
5486         struct instruction *ip = t->ip;
5487
5488         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5489
5490         JMP_CMP_MI(t, ip, <);
5491 }
5492
5493 static inline void
5494 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5495 {
5496         struct thread *t = &p->threads[p->thread_id];
5497         struct instruction *ip = t->ip;
5498
5499         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5500
5501         JMP_CMP_HI(t, ip, <);
5502 }
5503
5504 static inline void
5505 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5506 {
5507         struct thread *t = &p->threads[p->thread_id];
5508         struct instruction *ip = t->ip;
5509
5510         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5511
5512         JMP_CMP(t, ip, >);
5513 }
5514
5515 static inline void
5516 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5517 {
5518         struct thread *t = &p->threads[p->thread_id];
5519         struct instruction *ip = t->ip;
5520
5521         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5522
5523         JMP_CMP_MH(t, ip, >);
5524 }
5525
5526 static inline void
5527 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5528 {
5529         struct thread *t = &p->threads[p->thread_id];
5530         struct instruction *ip = t->ip;
5531
5532         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5533
5534         JMP_CMP_HM(t, ip, >);
5535 }
5536
5537 static inline void
5538 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5539 {
5540         struct thread *t = &p->threads[p->thread_id];
5541         struct instruction *ip = t->ip;
5542
5543         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5544
5545         JMP_CMP_HH(t, ip, >);
5546 }
5547
5548 static inline void
5549 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5550 {
5551         struct thread *t = &p->threads[p->thread_id];
5552         struct instruction *ip = t->ip;
5553
5554         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5555
5556         JMP_CMP_MI(t, ip, >);
5557 }
5558
5559 static inline void
5560 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5561 {
5562         struct thread *t = &p->threads[p->thread_id];
5563         struct instruction *ip = t->ip;
5564
5565         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5566
5567         JMP_CMP_HI(t, ip, >);
5568 }
5569
5570 /*
5571  * return.
5572  */
5573 static int
5574 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5575                        struct action *action,
5576                        char **tokens __rte_unused,
5577                        int n_tokens,
5578                        struct instruction *instr,
5579                        struct instruction_data *data __rte_unused)
5580 {
5581         CHECK(action, EINVAL);
5582         CHECK(n_tokens == 1, EINVAL);
5583
5584         instr->type = INSTR_RETURN;
5585         return 0;
5586 }
5587
5588 static inline void
5589 instr_return_exec(struct rte_swx_pipeline *p)
5590 {
5591         struct thread *t = &p->threads[p->thread_id];
5592
5593         TRACE("[Thread %2u] return\n", p->thread_id);
5594
5595         t->ip = t->ret;
5596 }
5597
5598 static int
5599 instr_translate(struct rte_swx_pipeline *p,
5600                 struct action *action,
5601                 char *string,
5602                 struct instruction *instr,
5603                 struct instruction_data *data)
5604 {
5605         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5606         int n_tokens = 0, tpos = 0;
5607
5608         /* Parse the instruction string into tokens. */
5609         for ( ; ; ) {
5610                 char *token;
5611
5612                 token = strtok_r(string, " \t\v", &string);
5613                 if (!token)
5614                         break;
5615
5616                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5617                 CHECK_NAME(token, EINVAL);
5618
5619                 tokens[n_tokens] = token;
5620                 n_tokens++;
5621         }
5622
5623         CHECK(n_tokens, EINVAL);
5624
5625         /* Handle the optional instruction label. */
5626         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5627                 strcpy(data->label, tokens[0]);
5628
5629                 tpos += 2;
5630                 CHECK(n_tokens - tpos, EINVAL);
5631         }
5632
5633         /* Identify the instruction type. */
5634         if (!strcmp(tokens[tpos], "rx"))
5635                 return instr_rx_translate(p,
5636                                           action,
5637                                           &tokens[tpos],
5638                                           n_tokens - tpos,
5639                                           instr,
5640                                           data);
5641
5642         if (!strcmp(tokens[tpos], "tx"))
5643                 return instr_tx_translate(p,
5644                                           action,
5645                                           &tokens[tpos],
5646                                           n_tokens - tpos,
5647                                           instr,
5648                                           data);
5649
5650         if (!strcmp(tokens[tpos], "drop"))
5651                 return instr_drop_translate(p,
5652                                             action,
5653                                             &tokens[tpos],
5654                                             n_tokens - tpos,
5655                                             instr,
5656                                             data);
5657
5658         if (!strcmp(tokens[tpos], "extract"))
5659                 return instr_hdr_extract_translate(p,
5660                                                    action,
5661                                                    &tokens[tpos],
5662                                                    n_tokens - tpos,
5663                                                    instr,
5664                                                    data);
5665
5666         if (!strcmp(tokens[tpos], "lookahead"))
5667                 return instr_hdr_lookahead_translate(p,
5668                                                      action,
5669                                                      &tokens[tpos],
5670                                                      n_tokens - tpos,
5671                                                      instr,
5672                                                      data);
5673
5674         if (!strcmp(tokens[tpos], "emit"))
5675                 return instr_hdr_emit_translate(p,
5676                                                 action,
5677                                                 &tokens[tpos],
5678                                                 n_tokens - tpos,
5679                                                 instr,
5680                                                 data);
5681
5682         if (!strcmp(tokens[tpos], "validate"))
5683                 return instr_hdr_validate_translate(p,
5684                                                     action,
5685                                                     &tokens[tpos],
5686                                                     n_tokens - tpos,
5687                                                     instr,
5688                                                     data);
5689
5690         if (!strcmp(tokens[tpos], "invalidate"))
5691                 return instr_hdr_invalidate_translate(p,
5692                                                       action,
5693                                                       &tokens[tpos],
5694                                                       n_tokens - tpos,
5695                                                       instr,
5696                                                       data);
5697
5698         if (!strcmp(tokens[tpos], "mov"))
5699                 return instr_mov_translate(p,
5700                                            action,
5701                                            &tokens[tpos],
5702                                            n_tokens - tpos,
5703                                            instr,
5704                                            data);
5705
5706         if (!strcmp(tokens[tpos], "add"))
5707                 return instr_alu_add_translate(p,
5708                                                action,
5709                                                &tokens[tpos],
5710                                                n_tokens - tpos,
5711                                                instr,
5712                                                data);
5713
5714         if (!strcmp(tokens[tpos], "sub"))
5715                 return instr_alu_sub_translate(p,
5716                                                action,
5717                                                &tokens[tpos],
5718                                                n_tokens - tpos,
5719                                                instr,
5720                                                data);
5721
5722         if (!strcmp(tokens[tpos], "ckadd"))
5723                 return instr_alu_ckadd_translate(p,
5724                                                  action,
5725                                                  &tokens[tpos],
5726                                                  n_tokens - tpos,
5727                                                  instr,
5728                                                  data);
5729
5730         if (!strcmp(tokens[tpos], "cksub"))
5731                 return instr_alu_cksub_translate(p,
5732                                                  action,
5733                                                  &tokens[tpos],
5734                                                  n_tokens - tpos,
5735                                                  instr,
5736                                                  data);
5737
5738         if (!strcmp(tokens[tpos], "and"))
5739                 return instr_alu_and_translate(p,
5740                                                action,
5741                                                &tokens[tpos],
5742                                                n_tokens - tpos,
5743                                                instr,
5744                                                data);
5745
5746         if (!strcmp(tokens[tpos], "or"))
5747                 return instr_alu_or_translate(p,
5748                                               action,
5749                                               &tokens[tpos],
5750                                               n_tokens - tpos,
5751                                               instr,
5752                                               data);
5753
5754         if (!strcmp(tokens[tpos], "xor"))
5755                 return instr_alu_xor_translate(p,
5756                                                action,
5757                                                &tokens[tpos],
5758                                                n_tokens - tpos,
5759                                                instr,
5760                                                data);
5761
5762         if (!strcmp(tokens[tpos], "shl"))
5763                 return instr_alu_shl_translate(p,
5764                                                action,
5765                                                &tokens[tpos],
5766                                                n_tokens - tpos,
5767                                                instr,
5768                                                data);
5769
5770         if (!strcmp(tokens[tpos], "shr"))
5771                 return instr_alu_shr_translate(p,
5772                                                action,
5773                                                &tokens[tpos],
5774                                                n_tokens - tpos,
5775                                                instr,
5776                                                data);
5777
5778         if (!strcmp(tokens[tpos], "regprefetch"))
5779                 return instr_regprefetch_translate(p,
5780                                                    action,
5781                                                    &tokens[tpos],
5782                                                    n_tokens - tpos,
5783                                                    instr,
5784                                                    data);
5785
5786         if (!strcmp(tokens[tpos], "regrd"))
5787                 return instr_regrd_translate(p,
5788                                              action,
5789                                              &tokens[tpos],
5790                                              n_tokens - tpos,
5791                                              instr,
5792                                              data);
5793
5794         if (!strcmp(tokens[tpos], "regwr"))
5795                 return instr_regwr_translate(p,
5796                                              action,
5797                                              &tokens[tpos],
5798                                              n_tokens - tpos,
5799                                              instr,
5800                                              data);
5801
5802         if (!strcmp(tokens[tpos], "regadd"))
5803                 return instr_regadd_translate(p,
5804                                               action,
5805                                               &tokens[tpos],
5806                                               n_tokens - tpos,
5807                                               instr,
5808                                               data);
5809
5810         if (!strcmp(tokens[tpos], "metprefetch"))
5811                 return instr_metprefetch_translate(p,
5812                                                    action,
5813                                                    &tokens[tpos],
5814                                                    n_tokens - tpos,
5815                                                    instr,
5816                                                    data);
5817
5818         if (!strcmp(tokens[tpos], "meter"))
5819                 return instr_meter_translate(p,
5820                                              action,
5821                                              &tokens[tpos],
5822                                              n_tokens - tpos,
5823                                              instr,
5824                                              data);
5825
5826         if (!strcmp(tokens[tpos], "table"))
5827                 return instr_table_translate(p,
5828                                              action,
5829                                              &tokens[tpos],
5830                                              n_tokens - tpos,
5831                                              instr,
5832                                              data);
5833
5834         if (!strcmp(tokens[tpos], "learn"))
5835                 return instr_learn_translate(p,
5836                                              action,
5837                                              &tokens[tpos],
5838                                              n_tokens - tpos,
5839                                              instr,
5840                                              data);
5841
5842         if (!strcmp(tokens[tpos], "forget"))
5843                 return instr_forget_translate(p,
5844                                               action,
5845                                               &tokens[tpos],
5846                                               n_tokens - tpos,
5847                                               instr,
5848                                               data);
5849
5850         if (!strcmp(tokens[tpos], "extern"))
5851                 return instr_extern_translate(p,
5852                                               action,
5853                                               &tokens[tpos],
5854                                               n_tokens - tpos,
5855                                               instr,
5856                                               data);
5857
5858         if (!strcmp(tokens[tpos], "jmp"))
5859                 return instr_jmp_translate(p,
5860                                            action,
5861                                            &tokens[tpos],
5862                                            n_tokens - tpos,
5863                                            instr,
5864                                            data);
5865
5866         if (!strcmp(tokens[tpos], "jmpv"))
5867                 return instr_jmp_valid_translate(p,
5868                                                  action,
5869                                                  &tokens[tpos],
5870                                                  n_tokens - tpos,
5871                                                  instr,
5872                                                  data);
5873
5874         if (!strcmp(tokens[tpos], "jmpnv"))
5875                 return instr_jmp_invalid_translate(p,
5876                                                    action,
5877                                                    &tokens[tpos],
5878                                                    n_tokens - tpos,
5879                                                    instr,
5880                                                    data);
5881
5882         if (!strcmp(tokens[tpos], "jmph"))
5883                 return instr_jmp_hit_translate(p,
5884                                                action,
5885                                                &tokens[tpos],
5886                                                n_tokens - tpos,
5887                                                instr,
5888                                                data);
5889
5890         if (!strcmp(tokens[tpos], "jmpnh"))
5891                 return instr_jmp_miss_translate(p,
5892                                                 action,
5893                                                 &tokens[tpos],
5894                                                 n_tokens - tpos,
5895                                                 instr,
5896                                                 data);
5897
5898         if (!strcmp(tokens[tpos], "jmpa"))
5899                 return instr_jmp_action_hit_translate(p,
5900                                                       action,
5901                                                       &tokens[tpos],
5902                                                       n_tokens - tpos,
5903                                                       instr,
5904                                                       data);
5905
5906         if (!strcmp(tokens[tpos], "jmpna"))
5907                 return instr_jmp_action_miss_translate(p,
5908                                                        action,
5909                                                        &tokens[tpos],
5910                                                        n_tokens - tpos,
5911                                                        instr,
5912                                                        data);
5913
5914         if (!strcmp(tokens[tpos], "jmpeq"))
5915                 return instr_jmp_eq_translate(p,
5916                                               action,
5917                                               &tokens[tpos],
5918                                               n_tokens - tpos,
5919                                               instr,
5920                                               data);
5921
5922         if (!strcmp(tokens[tpos], "jmpneq"))
5923                 return instr_jmp_neq_translate(p,
5924                                                action,
5925                                                &tokens[tpos],
5926                                                n_tokens - tpos,
5927                                                instr,
5928                                                data);
5929
5930         if (!strcmp(tokens[tpos], "jmplt"))
5931                 return instr_jmp_lt_translate(p,
5932                                               action,
5933                                               &tokens[tpos],
5934                                               n_tokens - tpos,
5935                                               instr,
5936                                               data);
5937
5938         if (!strcmp(tokens[tpos], "jmpgt"))
5939                 return instr_jmp_gt_translate(p,
5940                                               action,
5941                                               &tokens[tpos],
5942                                               n_tokens - tpos,
5943                                               instr,
5944                                               data);
5945
5946         if (!strcmp(tokens[tpos], "return"))
5947                 return instr_return_translate(p,
5948                                               action,
5949                                               &tokens[tpos],
5950                                               n_tokens - tpos,
5951                                               instr,
5952                                               data);
5953
5954         return -EINVAL;
5955 }
5956
5957 static struct instruction_data *
5958 label_find(struct instruction_data *data, uint32_t n, const char *label)
5959 {
5960         uint32_t i;
5961
5962         for (i = 0; i < n; i++)
5963                 if (!strcmp(label, data[i].label))
5964                         return &data[i];
5965
5966         return NULL;
5967 }
5968
5969 static uint32_t
5970 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5971 {
5972         uint32_t count = 0, i;
5973
5974         if (!label[0])
5975                 return 0;
5976
5977         for (i = 0; i < n; i++)
5978                 if (!strcmp(label, data[i].jmp_label))
5979                         count++;
5980
5981         return count;
5982 }
5983
5984 static int
5985 instr_label_check(struct instruction_data *instruction_data,
5986                   uint32_t n_instructions)
5987 {
5988         uint32_t i;
5989
5990         /* Check that all instruction labels are unique. */
5991         for (i = 0; i < n_instructions; i++) {
5992                 struct instruction_data *data = &instruction_data[i];
5993                 char *label = data->label;
5994                 uint32_t j;
5995
5996                 if (!label[0])
5997                         continue;
5998
5999                 for (j = i + 1; j < n_instructions; j++)
6000                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6001         }
6002
6003         /* Get users for each instruction label. */
6004         for (i = 0; i < n_instructions; i++) {
6005                 struct instruction_data *data = &instruction_data[i];
6006                 char *label = data->label;
6007
6008                 data->n_users = label_is_used(instruction_data,
6009                                               n_instructions,
6010                                               label);
6011         }
6012
6013         return 0;
6014 }
6015
6016 static int
6017 instr_jmp_resolve(struct instruction *instructions,
6018                   struct instruction_data *instruction_data,
6019                   uint32_t n_instructions)
6020 {
6021         uint32_t i;
6022
6023         for (i = 0; i < n_instructions; i++) {
6024                 struct instruction *instr = &instructions[i];
6025                 struct instruction_data *data = &instruction_data[i];
6026                 struct instruction_data *found;
6027
6028                 if (!instruction_is_jmp(instr))
6029                         continue;
6030
6031                 found = label_find(instruction_data,
6032                                    n_instructions,
6033                                    data->jmp_label);
6034                 CHECK(found, EINVAL);
6035
6036                 instr->jmp.ip = &instructions[found - instruction_data];
6037         }
6038
6039         return 0;
6040 }
6041
6042 static int
6043 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6044              struct action *a,
6045              struct instruction *instr,
6046              struct instruction_data *data __rte_unused,
6047              uint32_t n_instructions)
6048 {
6049         if (!a) {
6050                 enum instruction_type type;
6051                 uint32_t i;
6052
6053                 /* Check that the first instruction is rx. */
6054                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6055
6056                 /* Check that there is at least one tx instruction. */
6057                 for (i = 0; i < n_instructions; i++) {
6058                         type = instr[i].type;
6059
6060                         if (instruction_is_tx(type))
6061                                 break;
6062                 }
6063                 CHECK(i < n_instructions, EINVAL);
6064
6065                 /* Check that the last instruction is either tx or unconditional
6066                  * jump.
6067                  */
6068                 type = instr[n_instructions - 1].type;
6069                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6070         }
6071
6072         if (a) {
6073                 enum instruction_type type;
6074                 uint32_t i;
6075
6076                 /* Check that there is at least one return or tx instruction. */
6077                 for (i = 0; i < n_instructions; i++) {
6078                         type = instr[i].type;
6079
6080                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6081                                 break;
6082                 }
6083                 CHECK(i < n_instructions, EINVAL);
6084         }
6085
6086         return 0;
6087 }
6088
6089 static uint32_t
6090 instr_compact(struct instruction *instructions,
6091               struct instruction_data *instruction_data,
6092               uint32_t n_instructions)
6093 {
6094         uint32_t i, pos = 0;
6095
6096         /* Eliminate the invalid instructions that have been optimized out. */
6097         for (i = 0; i < n_instructions; i++) {
6098                 struct instruction *instr = &instructions[i];
6099                 struct instruction_data *data = &instruction_data[i];
6100
6101                 if (data->invalid)
6102                         continue;
6103
6104                 if (i != pos) {
6105                         memcpy(&instructions[pos], instr, sizeof(*instr));
6106                         memcpy(&instruction_data[pos], data, sizeof(*data));
6107                 }
6108
6109                 pos++;
6110         }
6111
6112         return pos;
6113 }
6114
6115 static int
6116 instr_pattern_extract_many_search(struct instruction *instr,
6117                                   struct instruction_data *data,
6118                                   uint32_t n_instr,
6119                                   uint32_t *n_pattern_instr)
6120 {
6121         uint32_t i;
6122
6123         for (i = 0; i < n_instr; i++) {
6124                 if (data[i].invalid)
6125                         break;
6126
6127                 if (instr[i].type != INSTR_HDR_EXTRACT)
6128                         break;
6129
6130                 if (i == RTE_DIM(instr->io.hdr.header_id))
6131                         break;
6132
6133                 if (i && data[i].n_users)
6134                         break;
6135         }
6136
6137         if (i < 2)
6138                 return 0;
6139
6140         *n_pattern_instr = i;
6141         return 1;
6142 }
6143
6144 static void
6145 instr_pattern_extract_many_replace(struct instruction *instr,
6146                                    struct instruction_data *data,
6147                                    uint32_t n_instr)
6148 {
6149         uint32_t i;
6150
6151         for (i = 1; i < n_instr; i++) {
6152                 instr[0].type++;
6153                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6154                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6155                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6156
6157                 data[i].invalid = 1;
6158         }
6159 }
6160
6161 static uint32_t
6162 instr_pattern_extract_many_optimize(struct instruction *instructions,
6163                                     struct instruction_data *instruction_data,
6164                                     uint32_t n_instructions)
6165 {
6166         uint32_t i;
6167
6168         for (i = 0; i < n_instructions; ) {
6169                 struct instruction *instr = &instructions[i];
6170                 struct instruction_data *data = &instruction_data[i];
6171                 uint32_t n_instr = 0;
6172                 int detected;
6173
6174                 /* Extract many. */
6175                 detected = instr_pattern_extract_many_search(instr,
6176                                                              data,
6177                                                              n_instructions - i,
6178                                                              &n_instr);
6179                 if (detected) {
6180                         instr_pattern_extract_many_replace(instr,
6181                                                            data,
6182                                                            n_instr);
6183                         i += n_instr;
6184                         continue;
6185                 }
6186
6187                 /* No pattern starting at the current instruction. */
6188                 i++;
6189         }
6190
6191         /* Eliminate the invalid instructions that have been optimized out. */
6192         n_instructions = instr_compact(instructions,
6193                                        instruction_data,
6194                                        n_instructions);
6195
6196         return n_instructions;
6197 }
6198
6199 static int
6200 instr_pattern_emit_many_tx_search(struct instruction *instr,
6201                                   struct instruction_data *data,
6202                                   uint32_t n_instr,
6203                                   uint32_t *n_pattern_instr)
6204 {
6205         uint32_t i;
6206
6207         for (i = 0; i < n_instr; i++) {
6208                 if (data[i].invalid)
6209                         break;
6210
6211                 if (instr[i].type != INSTR_HDR_EMIT)
6212                         break;
6213
6214                 if (i == RTE_DIM(instr->io.hdr.header_id))
6215                         break;
6216
6217                 if (i && data[i].n_users)
6218                         break;
6219         }
6220
6221         if (!i)
6222                 return 0;
6223
6224         if (instr[i].type != INSTR_TX)
6225                 return 0;
6226
6227         if (data[i].n_users)
6228                 return 0;
6229
6230         i++;
6231
6232         *n_pattern_instr = i;
6233         return 1;
6234 }
6235
6236 static void
6237 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6238                                    struct instruction_data *data,
6239                                    uint32_t n_instr)
6240 {
6241         uint32_t i;
6242
6243         /* Any emit instruction in addition to the first one. */
6244         for (i = 1; i < n_instr - 1; i++) {
6245                 instr[0].type++;
6246                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6247                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6248                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6249
6250                 data[i].invalid = 1;
6251         }
6252
6253         /* The TX instruction is the last one in the pattern. */
6254         instr[0].type++;
6255         instr[0].io.io.offset = instr[i].io.io.offset;
6256         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6257         data[i].invalid = 1;
6258 }
6259
6260 static uint32_t
6261 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6262                                     struct instruction_data *instruction_data,
6263                                     uint32_t n_instructions)
6264 {
6265         uint32_t i;
6266
6267         for (i = 0; i < n_instructions; ) {
6268                 struct instruction *instr = &instructions[i];
6269                 struct instruction_data *data = &instruction_data[i];
6270                 uint32_t n_instr = 0;
6271                 int detected;
6272
6273                 /* Emit many + TX. */
6274                 detected = instr_pattern_emit_many_tx_search(instr,
6275                                                              data,
6276                                                              n_instructions - i,
6277                                                              &n_instr);
6278                 if (detected) {
6279                         instr_pattern_emit_many_tx_replace(instr,
6280                                                            data,
6281                                                            n_instr);
6282                         i += n_instr;
6283                         continue;
6284                 }
6285
6286                 /* No pattern starting at the current instruction. */
6287                 i++;
6288         }
6289
6290         /* Eliminate the invalid instructions that have been optimized out. */
6291         n_instructions = instr_compact(instructions,
6292                                        instruction_data,
6293                                        n_instructions);
6294
6295         return n_instructions;
6296 }
6297
6298 static uint32_t
6299 action_arg_src_mov_count(struct action *a,
6300                          uint32_t arg_id,
6301                          struct instruction *instructions,
6302                          struct instruction_data *instruction_data,
6303                          uint32_t n_instructions);
6304
6305 static int
6306 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6307                                       struct action *a,
6308                                       struct instruction *instr,
6309                                       struct instruction_data *data,
6310                                       uint32_t n_instr,
6311                                       struct instruction *instructions,
6312                                       struct instruction_data *instruction_data,
6313                                       uint32_t n_instructions,
6314                                       uint32_t *n_pattern_instr)
6315 {
6316         struct header *h;
6317         uint32_t src_field_id, i, j;
6318
6319         /* Prerequisites. */
6320         if (!a || !a->st)
6321                 return 0;
6322
6323         /* First instruction: MOV_HM. */
6324         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6325                 return 0;
6326
6327         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6328         if (!h || h->st->var_size)
6329                 return 0;
6330
6331         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6332                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6333                         break;
6334
6335         if (src_field_id == a->st->n_fields)
6336                 return 0;
6337
6338         if (instr[0].mov.dst.offset ||
6339             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6340             instr[0].mov.src.struct_id ||
6341             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6342             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6343                 return 0;
6344
6345         if ((n_instr < h->st->n_fields + 1) ||
6346              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6347                 return 0;
6348
6349         /* Subsequent instructions: MOV_HM. */
6350         for (i = 1; i < h->st->n_fields; i++)
6351                 if (data[i].invalid ||
6352                     data[i].n_users ||
6353                     (instr[i].type != INSTR_MOV_HM) ||
6354                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6355                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6356                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6357                     instr[i].mov.src.struct_id ||
6358                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6359                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6360                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6361                         return 0;
6362
6363         /* Last instruction: HDR_VALIDATE. */
6364         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6365             (instr[i].valid.header_id != h->id))
6366                 return 0;
6367
6368         /* Check that none of the action args that are used as source for this
6369          * DMA transfer are not used as source in any other mov instruction.
6370          */
6371         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6372                 uint32_t n_users;
6373
6374                 n_users = action_arg_src_mov_count(a,
6375                                                    j,
6376                                                    instructions,
6377                                                    instruction_data,
6378                                                    n_instructions);
6379                 if (n_users > 1)
6380                         return 0;
6381         }
6382
6383         *n_pattern_instr = 1 + i;
6384         return 1;
6385 }
6386
6387 static void
6388 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6389                                        struct action *a,
6390                                        struct instruction *instr,
6391                                        struct instruction_data *data,
6392                                        uint32_t n_instr)
6393 {
6394         struct header *h;
6395         uint32_t src_field_id, src_offset, i;
6396
6397         /* Read from the instructions before they are modified. */
6398         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6399         if (!h)
6400                 return;
6401
6402         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6403                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6404                         break;
6405
6406         if (src_field_id == a->st->n_fields)
6407                 return;
6408
6409         src_offset = instr[0].mov.src.offset;
6410
6411         /* Modify the instructions. */
6412         instr[0].type = INSTR_DMA_HT;
6413         instr[0].dma.dst.header_id[0] = h->id;
6414         instr[0].dma.dst.struct_id[0] = h->struct_id;
6415         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6416         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6417
6418         for (i = 1; i < n_instr; i++)
6419                 data[i].invalid = 1;
6420
6421         /* Update the endianness of the action arguments to header endianness. */
6422         for (i = 0; i < h->st->n_fields; i++)
6423                 a->args_endianness[src_field_id + i] = 1;
6424 }
6425
6426 static uint32_t
6427 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6428                                         struct action *a,
6429                                         struct instruction *instructions,
6430                                         struct instruction_data *instruction_data,
6431                                         uint32_t n_instructions)
6432 {
6433         uint32_t i;
6434
6435         if (!a || !a->st)
6436                 return n_instructions;
6437
6438         for (i = 0; i < n_instructions; ) {
6439                 struct instruction *instr = &instructions[i];
6440                 struct instruction_data *data = &instruction_data[i];
6441                 uint32_t n_instr = 0;
6442                 int detected;
6443
6444                 /* Mov all + validate. */
6445                 detected = instr_pattern_mov_all_validate_search(p,
6446                                                                  a,
6447                                                                  instr,
6448                                                                  data,
6449                                                                  n_instructions - i,
6450                                                                  instructions,
6451                                                                  instruction_data,
6452                                                                  n_instructions,
6453                                                                  &n_instr);
6454                 if (detected) {
6455                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6456                         i += n_instr;
6457                         continue;
6458                 }
6459
6460                 /* No pattern starting at the current instruction. */
6461                 i++;
6462         }
6463
6464         /* Eliminate the invalid instructions that have been optimized out. */
6465         n_instructions = instr_compact(instructions,
6466                                        instruction_data,
6467                                        n_instructions);
6468
6469         return n_instructions;
6470 }
6471
6472 static int
6473 instr_pattern_dma_many_search(struct instruction *instr,
6474                               struct instruction_data *data,
6475                               uint32_t n_instr,
6476                               uint32_t *n_pattern_instr)
6477 {
6478         uint32_t i;
6479
6480         for (i = 0; i < n_instr; i++) {
6481                 if (data[i].invalid)
6482                         break;
6483
6484                 if (instr[i].type != INSTR_DMA_HT)
6485                         break;
6486
6487                 if (i == RTE_DIM(instr->dma.dst.header_id))
6488                         break;
6489
6490                 if (i && data[i].n_users)
6491                         break;
6492         }
6493
6494         if (i < 2)
6495                 return 0;
6496
6497         *n_pattern_instr = i;
6498         return 1;
6499 }
6500
6501 static void
6502 instr_pattern_dma_many_replace(struct instruction *instr,
6503                                struct instruction_data *data,
6504                                uint32_t n_instr)
6505 {
6506         uint32_t i;
6507
6508         for (i = 1; i < n_instr; i++) {
6509                 instr[0].type++;
6510                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6511                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6512                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6513                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6514
6515                 data[i].invalid = 1;
6516         }
6517 }
6518
6519 static uint32_t
6520 instr_pattern_dma_many_optimize(struct instruction *instructions,
6521                struct instruction_data *instruction_data,
6522                uint32_t n_instructions)
6523 {
6524         uint32_t i;
6525
6526         for (i = 0; i < n_instructions; ) {
6527                 struct instruction *instr = &instructions[i];
6528                 struct instruction_data *data = &instruction_data[i];
6529                 uint32_t n_instr = 0;
6530                 int detected;
6531
6532                 /* DMA many. */
6533                 detected = instr_pattern_dma_many_search(instr,
6534                                                          data,
6535                                                          n_instructions - i,
6536                                                          &n_instr);
6537                 if (detected) {
6538                         instr_pattern_dma_many_replace(instr, data, n_instr);
6539                         i += n_instr;
6540                         continue;
6541                 }
6542
6543                 /* No pattern starting at the current instruction. */
6544                 i++;
6545         }
6546
6547         /* Eliminate the invalid instructions that have been optimized out. */
6548         n_instructions = instr_compact(instructions,
6549                                        instruction_data,
6550                                        n_instructions);
6551
6552         return n_instructions;
6553 }
6554
6555 static uint32_t
6556 instr_optimize(struct rte_swx_pipeline *p,
6557                struct action *a,
6558                struct instruction *instructions,
6559                struct instruction_data *instruction_data,
6560                uint32_t n_instructions)
6561 {
6562         /* Extract many. */
6563         n_instructions = instr_pattern_extract_many_optimize(instructions,
6564                                                              instruction_data,
6565                                                              n_instructions);
6566
6567         /* Emit many + TX. */
6568         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6569                                                              instruction_data,
6570                                                              n_instructions);
6571
6572         /* Mov all + validate. */
6573         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6574                                                                  a,
6575                                                                  instructions,
6576                                                                  instruction_data,
6577                                                                  n_instructions);
6578
6579         /* DMA many. */
6580         n_instructions = instr_pattern_dma_many_optimize(instructions,
6581                                                          instruction_data,
6582                                                          n_instructions);
6583
6584         return n_instructions;
6585 }
6586
6587 static int
6588 instruction_config(struct rte_swx_pipeline *p,
6589                    struct action *a,
6590                    const char **instructions,
6591                    uint32_t n_instructions)
6592 {
6593         struct instruction *instr = NULL;
6594         struct instruction_data *data = NULL;
6595         int err = 0;
6596         uint32_t i;
6597
6598         CHECK(n_instructions, EINVAL);
6599         CHECK(instructions, EINVAL);
6600         for (i = 0; i < n_instructions; i++)
6601                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6602
6603         /* Memory allocation. */
6604         instr = calloc(n_instructions, sizeof(struct instruction));
6605         if (!instr) {
6606                 err = -ENOMEM;
6607                 goto error;
6608         }
6609
6610         data = calloc(n_instructions, sizeof(struct instruction_data));
6611         if (!data) {
6612                 err = -ENOMEM;
6613                 goto error;
6614         }
6615
6616         for (i = 0; i < n_instructions; i++) {
6617                 char *string = strdup(instructions[i]);
6618                 if (!string) {
6619                         err = -ENOMEM;
6620                         goto error;
6621                 }
6622
6623                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6624                 if (err) {
6625                         free(string);
6626                         goto error;
6627                 }
6628
6629                 free(string);
6630         }
6631
6632         err = instr_label_check(data, n_instructions);
6633         if (err)
6634                 goto error;
6635
6636         err = instr_verify(p, a, instr, data, n_instructions);
6637         if (err)
6638                 goto error;
6639
6640         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6641
6642         err = instr_jmp_resolve(instr, data, n_instructions);
6643         if (err)
6644                 goto error;
6645
6646         if (a) {
6647                 a->instructions = instr;
6648                 a->instruction_data = data;
6649                 a->n_instructions = n_instructions;
6650         } else {
6651                 p->instructions = instr;
6652                 p->instruction_data = data;
6653                 p->n_instructions = n_instructions;
6654         }
6655
6656         return 0;
6657
6658 error:
6659         free(data);
6660         free(instr);
6661         return err;
6662 }
6663
6664 static instr_exec_t instruction_table[] = {
6665         [INSTR_RX] = instr_rx_exec,
6666         [INSTR_TX] = instr_tx_exec,
6667         [INSTR_TX_I] = instr_tx_i_exec,
6668         [INSTR_DROP] = instr_drop_exec,
6669
6670         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6671         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6672         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6673         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6674         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6675         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6676         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6677         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6678         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6679         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6680
6681         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6682         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6683         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6684         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6685         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6686         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6687         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6688         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6689         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6690
6691         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6692         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6693
6694         [INSTR_MOV] = instr_mov_exec,
6695         [INSTR_MOV_MH] = instr_mov_mh_exec,
6696         [INSTR_MOV_HM] = instr_mov_hm_exec,
6697         [INSTR_MOV_HH] = instr_mov_hh_exec,
6698         [INSTR_MOV_I] = instr_mov_i_exec,
6699
6700         [INSTR_DMA_HT] = instr_dma_ht_exec,
6701         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6702         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6703         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6704         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6705         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6706         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6707         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6708
6709         [INSTR_ALU_ADD] = instr_alu_add_exec,
6710         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6711         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6712         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6713         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6714         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6715
6716         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6717         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6718         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6719         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6720         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6721         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6722
6723         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6724         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6725         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6726         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6727
6728         [INSTR_ALU_AND] = instr_alu_and_exec,
6729         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6730         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6731         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6732         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6733
6734         [INSTR_ALU_OR] = instr_alu_or_exec,
6735         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6736         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6737         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6738         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6739
6740         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6741         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6742         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6743         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6744         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6745
6746         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6747         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6748         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6749         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6750         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6751         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6752
6753         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6754         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6755         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6756         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6757         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6758         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6759
6760         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6761         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6762         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6763
6764         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6765         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6766         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6767         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6768         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6769         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6770
6771         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6772         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6773         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6774         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6775         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6776         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6777         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6778         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6779         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6780
6781         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6782         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6783         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6784         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6785         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6786         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6787         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6788         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6789         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6790
6791         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6792         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6793         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6794
6795         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6796         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6797         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6798         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6799         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6800         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6801         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6802         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6803         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6804         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6805         [INSTR_METER_IMM] = instr_meter_imm_exec,
6806         [INSTR_METER_IMI] = instr_meter_imi_exec,
6807
6808         [INSTR_TABLE] = instr_table_exec,
6809         [INSTR_TABLE_AF] = instr_table_af_exec,
6810         [INSTR_SELECTOR] = instr_selector_exec,
6811         [INSTR_LEARNER] = instr_learner_exec,
6812         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6813         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6814         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6815         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6816         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6817
6818         [INSTR_JMP] = instr_jmp_exec,
6819         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6820         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6821         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6822         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6823         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6824         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6825
6826         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6827         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6828         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6829         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6830         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6831
6832         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6833         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6834         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6835         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6836         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6837
6838         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6839         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6840         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6841         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6842         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6843         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6844
6845         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6846         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6847         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6848         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6849         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6850         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6851
6852         [INSTR_RETURN] = instr_return_exec,
6853 };
6854
6855 static int
6856 instruction_table_build(struct rte_swx_pipeline *p)
6857 {
6858         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6859                                       sizeof(struct instr_exec_t *));
6860         if (!p->instruction_table)
6861                 return -EINVAL;
6862
6863         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6864
6865         return 0;
6866 }
6867
6868 static void
6869 instruction_table_build_free(struct rte_swx_pipeline *p)
6870 {
6871         if (!p->instruction_table)
6872                 return;
6873
6874         free(p->instruction_table);
6875         p->instruction_table = NULL;
6876 }
6877
6878 static void
6879 instruction_table_free(struct rte_swx_pipeline *p)
6880 {
6881         instruction_table_build_free(p);
6882 }
6883
6884 static inline void
6885 instr_exec(struct rte_swx_pipeline *p)
6886 {
6887         struct thread *t = &p->threads[p->thread_id];
6888         struct instruction *ip = t->ip;
6889         instr_exec_t instr = p->instruction_table[ip->type];
6890
6891         instr(p);
6892 }
6893
6894 /*
6895  * Action.
6896  */
6897 static struct action *
6898 action_find(struct rte_swx_pipeline *p, const char *name)
6899 {
6900         struct action *elem;
6901
6902         if (!name)
6903                 return NULL;
6904
6905         TAILQ_FOREACH(elem, &p->actions, node)
6906                 if (strcmp(elem->name, name) == 0)
6907                         return elem;
6908
6909         return NULL;
6910 }
6911
6912 static struct action *
6913 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6914 {
6915         struct action *action = NULL;
6916
6917         TAILQ_FOREACH(action, &p->actions, node)
6918                 if (action->id == id)
6919                         return action;
6920
6921         return NULL;
6922 }
6923
6924 static struct field *
6925 action_field_find(struct action *a, const char *name)
6926 {
6927         return a->st ? struct_type_field_find(a->st, name) : NULL;
6928 }
6929
6930 static struct field *
6931 action_field_parse(struct action *action, const char *name)
6932 {
6933         if (name[0] != 't' || name[1] != '.')
6934                 return NULL;
6935
6936         return action_field_find(action, &name[2]);
6937 }
6938
6939 static int
6940 action_has_nbo_args(struct action *a)
6941 {
6942         uint32_t i;
6943
6944         /* Return if the action does not have any args. */
6945         if (!a->st)
6946                 return 0; /* FALSE */
6947
6948         for (i = 0; i < a->st->n_fields; i++)
6949                 if (a->args_endianness[i])
6950                         return 1; /* TRUE */
6951
6952         return 0; /* FALSE */
6953 }
6954
6955 static int
6956 action_does_learning(struct action *a)
6957 {
6958         uint32_t i;
6959
6960         for (i = 0; i < a->n_instructions; i++)
6961                 switch (a->instructions[i].type) {
6962                 case INSTR_LEARNER_LEARN:
6963                         return 1; /* TRUE */
6964
6965                 case INSTR_LEARNER_FORGET:
6966                         return 1; /* TRUE */
6967
6968                 default:
6969                         continue;
6970                 }
6971
6972         return 0; /* FALSE */
6973 }
6974
6975 int
6976 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6977                                const char *name,
6978                                const char *args_struct_type_name,
6979                                const char **instructions,
6980                                uint32_t n_instructions)
6981 {
6982         struct struct_type *args_struct_type = NULL;
6983         struct action *a;
6984         int err;
6985
6986         CHECK(p, EINVAL);
6987
6988         CHECK_NAME(name, EINVAL);
6989         CHECK(!action_find(p, name), EEXIST);
6990
6991         if (args_struct_type_name) {
6992                 CHECK_NAME(args_struct_type_name, EINVAL);
6993                 args_struct_type = struct_type_find(p, args_struct_type_name);
6994                 CHECK(args_struct_type, EINVAL);
6995                 CHECK(!args_struct_type->var_size, EINVAL);
6996         }
6997
6998         /* Node allocation. */
6999         a = calloc(1, sizeof(struct action));
7000         CHECK(a, ENOMEM);
7001         if (args_struct_type) {
7002                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7003                 if (!a->args_endianness) {
7004                         free(a);
7005                         CHECK(0, ENOMEM);
7006                 }
7007         }
7008
7009         /* Node initialization. */
7010         strcpy(a->name, name);
7011         a->st = args_struct_type;
7012         a->id = p->n_actions;
7013
7014         /* Instruction translation. */
7015         err = instruction_config(p, a, instructions, n_instructions);
7016         if (err) {
7017                 free(a->args_endianness);
7018                 free(a);
7019                 return err;
7020         }
7021
7022         /* Node add to tailq. */
7023         TAILQ_INSERT_TAIL(&p->actions, a, node);
7024         p->n_actions++;
7025
7026         return 0;
7027 }
7028
7029 static int
7030 action_build(struct rte_swx_pipeline *p)
7031 {
7032         struct action *action;
7033
7034         /* p->action_instructions. */
7035         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7036         CHECK(p->action_instructions, ENOMEM);
7037
7038         TAILQ_FOREACH(action, &p->actions, node)
7039                 p->action_instructions[action->id] = action->instructions;
7040
7041         /* p->action_funcs. */
7042         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7043         CHECK(p->action_funcs, ENOMEM);
7044
7045         return 0;
7046 }
7047
7048 static void
7049 action_build_free(struct rte_swx_pipeline *p)
7050 {
7051         free(p->action_funcs);
7052         p->action_funcs = NULL;
7053
7054         free(p->action_instructions);
7055         p->action_instructions = NULL;
7056 }
7057
7058 static void
7059 action_free(struct rte_swx_pipeline *p)
7060 {
7061         action_build_free(p);
7062
7063         for ( ; ; ) {
7064                 struct action *action;
7065
7066                 action = TAILQ_FIRST(&p->actions);
7067                 if (!action)
7068                         break;
7069
7070                 TAILQ_REMOVE(&p->actions, action, node);
7071                 free(action->instruction_data);
7072                 free(action->instructions);
7073                 free(action);
7074         }
7075 }
7076
7077 static uint32_t
7078 action_arg_src_mov_count(struct action *a,
7079                          uint32_t arg_id,
7080                          struct instruction *instructions,
7081                          struct instruction_data *instruction_data,
7082                          uint32_t n_instructions)
7083 {
7084         uint32_t offset, n_users = 0, i;
7085
7086         if (!a->st ||
7087             (arg_id >= a->st->n_fields) ||
7088             !instructions ||
7089             !instruction_data ||
7090             !n_instructions)
7091                 return 0;
7092
7093         offset = a->st->fields[arg_id].offset / 8;
7094
7095         for (i = 0; i < n_instructions; i++) {
7096                 struct instruction *instr = &instructions[i];
7097                 struct instruction_data *data = &instruction_data[i];
7098
7099                 if (data->invalid ||
7100                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7101                     instr->mov.src.struct_id ||
7102                     (instr->mov.src.offset != offset))
7103                         continue;
7104
7105                 n_users++;
7106         }
7107
7108         return n_users;
7109 }
7110
7111 /*
7112  * Table.
7113  */
7114 static struct table_type *
7115 table_type_find(struct rte_swx_pipeline *p, const char *name)
7116 {
7117         struct table_type *elem;
7118
7119         TAILQ_FOREACH(elem, &p->table_types, node)
7120                 if (strcmp(elem->name, name) == 0)
7121                         return elem;
7122
7123         return NULL;
7124 }
7125
7126 static struct table_type *
7127 table_type_resolve(struct rte_swx_pipeline *p,
7128                    const char *recommended_type_name,
7129                    enum rte_swx_table_match_type match_type)
7130 {
7131         struct table_type *elem;
7132
7133         /* Only consider the recommended type if the match type is correct. */
7134         if (recommended_type_name)
7135                 TAILQ_FOREACH(elem, &p->table_types, node)
7136                         if (!strcmp(elem->name, recommended_type_name) &&
7137                             (elem->match_type == match_type))
7138                                 return elem;
7139
7140         /* Ignore the recommended type and get the first element with this match
7141          * type.
7142          */
7143         TAILQ_FOREACH(elem, &p->table_types, node)
7144                 if (elem->match_type == match_type)
7145                         return elem;
7146
7147         return NULL;
7148 }
7149
7150 static struct table *
7151 table_find(struct rte_swx_pipeline *p, const char *name)
7152 {
7153         struct table *elem;
7154
7155         TAILQ_FOREACH(elem, &p->tables, node)
7156                 if (strcmp(elem->name, name) == 0)
7157                         return elem;
7158
7159         return NULL;
7160 }
7161
7162 static struct table *
7163 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7164 {
7165         struct table *table = NULL;
7166
7167         TAILQ_FOREACH(table, &p->tables, node)
7168                 if (table->id == id)
7169                         return table;
7170
7171         return NULL;
7172 }
7173
7174 int
7175 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7176                                      const char *name,
7177                                      enum rte_swx_table_match_type match_type,
7178                                      struct rte_swx_table_ops *ops)
7179 {
7180         struct table_type *elem;
7181
7182         CHECK(p, EINVAL);
7183
7184         CHECK_NAME(name, EINVAL);
7185         CHECK(!table_type_find(p, name), EEXIST);
7186
7187         CHECK(ops, EINVAL);
7188         CHECK(ops->create, EINVAL);
7189         CHECK(ops->lkp, EINVAL);
7190         CHECK(ops->free, EINVAL);
7191
7192         /* Node allocation. */
7193         elem = calloc(1, sizeof(struct table_type));
7194         CHECK(elem, ENOMEM);
7195
7196         /* Node initialization. */
7197         strcpy(elem->name, name);
7198         elem->match_type = match_type;
7199         memcpy(&elem->ops, ops, sizeof(*ops));
7200
7201         /* Node add to tailq. */
7202         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7203
7204         return 0;
7205 }
7206
7207 static int
7208 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7209                          uint32_t n_fields,
7210                          enum rte_swx_table_match_type *match_type)
7211 {
7212         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7213
7214         for (i = 0; i < n_fields; i++) {
7215                 struct rte_swx_match_field_params  *f = &fields[i];
7216
7217                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7218                         n_fields_em++;
7219
7220                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7221                         n_fields_lpm++;
7222         }
7223
7224         if ((n_fields_lpm > 1) ||
7225             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7226                 return -EINVAL;
7227
7228         *match_type = (n_fields_em == n_fields) ?
7229                        RTE_SWX_TABLE_MATCH_EXACT :
7230                        RTE_SWX_TABLE_MATCH_WILDCARD;
7231
7232         return 0;
7233 }
7234
7235 static int
7236 table_match_fields_check(struct rte_swx_pipeline *p,
7237                          struct rte_swx_pipeline_table_params *params,
7238                          struct header **header)
7239 {
7240         struct header *h0 = NULL;
7241         struct field *hf, *mf;
7242         uint32_t *offset = NULL, i;
7243         int status = 0;
7244
7245         /* Return if no match fields. */
7246         if (!params->n_fields) {
7247                 if (params->fields) {
7248                         status = -EINVAL;
7249                         goto end;
7250                 }
7251
7252                 if (header)
7253                         *header = NULL;
7254
7255                 return 0;
7256         }
7257
7258         /* Memory allocation. */
7259         offset = calloc(params->n_fields, sizeof(uint32_t));
7260         if (!offset) {
7261                 status = -ENOMEM;
7262                 goto end;
7263         }
7264
7265         /* Check that all the match fields belong to either the same header or
7266          * to the meta-data.
7267          */
7268         hf = header_field_parse(p, params->fields[0].name, &h0);
7269         mf = metadata_field_parse(p, params->fields[0].name);
7270         if ((!hf && !mf) || (hf && hf->var_size)) {
7271                 status = -EINVAL;
7272                 goto end;
7273         }
7274
7275         offset[0] = h0 ? hf->offset : mf->offset;
7276
7277         for (i = 1; i < params->n_fields; i++)
7278                 if (h0) {
7279                         struct header *h;
7280
7281                         hf = header_field_parse(p, params->fields[i].name, &h);
7282                         if (!hf || (h->id != h0->id) || hf->var_size) {
7283                                 status = -EINVAL;
7284                                 goto end;
7285                         }
7286
7287                         offset[i] = hf->offset;
7288                 } else {
7289                         mf = metadata_field_parse(p, params->fields[i].name);
7290                         if (!mf) {
7291                                 status = -EINVAL;
7292                                 goto end;
7293                         }
7294
7295                         offset[i] = mf->offset;
7296                 }
7297
7298         /* Check that there are no duplicated match fields. */
7299         for (i = 0; i < params->n_fields; i++) {
7300                 uint32_t j;
7301
7302                 for (j = 0; j < i; j++)
7303                         if (offset[j] == offset[i]) {
7304                                 status = -EINVAL;
7305                                 goto end;
7306                         }
7307         }
7308
7309         /* Return. */
7310         if (header)
7311                 *header = h0;
7312
7313 end:
7314         free(offset);
7315         return status;
7316 }
7317
7318 int
7319 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7320                               const char *name,
7321                               struct rte_swx_pipeline_table_params *params,
7322                               const char *recommended_table_type_name,
7323                               const char *args,
7324                               uint32_t size)
7325 {
7326         struct table_type *type;
7327         struct table *t = NULL;
7328         struct action *default_action;
7329         struct header *header = NULL;
7330         uint32_t action_data_size_max = 0, i;
7331         int status = 0;
7332
7333         CHECK(p, EINVAL);
7334
7335         CHECK_NAME(name, EINVAL);
7336         CHECK(!table_find(p, name), EEXIST);
7337         CHECK(!selector_find(p, name), EEXIST);
7338         CHECK(!learner_find(p, name), EEXIST);
7339
7340         CHECK(params, EINVAL);
7341
7342         /* Match checks. */
7343         status = table_match_fields_check(p, params, &header);
7344         if (status)
7345                 return status;
7346
7347         /* Action checks. */
7348         CHECK(params->n_actions, EINVAL);
7349         CHECK(params->action_names, EINVAL);
7350         for (i = 0; i < params->n_actions; i++) {
7351                 const char *action_name = params->action_names[i];
7352                 struct action *a;
7353                 uint32_t action_data_size;
7354                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7355
7356                 CHECK_NAME(action_name, EINVAL);
7357
7358                 a = action_find(p, action_name);
7359                 CHECK(a, EINVAL);
7360                 CHECK(!action_does_learning(a), EINVAL);
7361
7362                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7363                 if (action_data_size > action_data_size_max)
7364                         action_data_size_max = action_data_size;
7365
7366                 if (params->action_is_for_table_entries)
7367                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7368                 if (params->action_is_for_default_entry)
7369                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7370                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7371         }
7372
7373         CHECK_NAME(params->default_action_name, EINVAL);
7374         for (i = 0; i < p->n_actions; i++)
7375                 if (!strcmp(params->action_names[i],
7376                             params->default_action_name))
7377                         break;
7378         CHECK(i < params->n_actions, EINVAL);
7379         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7380               EINVAL);
7381
7382         default_action = action_find(p, params->default_action_name);
7383         CHECK((default_action->st && params->default_action_data) ||
7384               !params->default_action_data, EINVAL);
7385
7386         /* Table type checks. */
7387         if (recommended_table_type_name)
7388                 CHECK_NAME(recommended_table_type_name, EINVAL);
7389
7390         if (params->n_fields) {
7391                 enum rte_swx_table_match_type match_type;
7392
7393                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7394                 if (status)
7395                         return status;
7396
7397                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7398                 CHECK(type, EINVAL);
7399         } else {
7400                 type = NULL;
7401         }
7402
7403         /* Memory allocation. */
7404         t = calloc(1, sizeof(struct table));
7405         if (!t)
7406                 goto nomem;
7407
7408         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7409         if (!t->fields)
7410                 goto nomem;
7411
7412         t->actions = calloc(params->n_actions, sizeof(struct action *));
7413         if (!t->actions)
7414                 goto nomem;
7415
7416         if (action_data_size_max) {
7417                 t->default_action_data = calloc(1, action_data_size_max);
7418                 if (!t->default_action_data)
7419                         goto nomem;
7420         }
7421
7422         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7423         if (!t->action_is_for_table_entries)
7424                 goto nomem;
7425
7426         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7427         if (!t->action_is_for_default_entry)
7428                 goto nomem;
7429
7430         /* Node initialization. */
7431         strcpy(t->name, name);
7432         if (args && args[0])
7433                 strcpy(t->args, args);
7434         t->type = type;
7435
7436         for (i = 0; i < params->n_fields; i++) {
7437                 struct rte_swx_match_field_params *field = &params->fields[i];
7438                 struct match_field *f = &t->fields[i];
7439
7440                 f->match_type = field->match_type;
7441                 f->field = header ?
7442                         header_field_parse(p, field->name, NULL) :
7443                         metadata_field_parse(p, field->name);
7444         }
7445         t->n_fields = params->n_fields;
7446         t->header = header;
7447
7448         for (i = 0; i < params->n_actions; i++) {
7449                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7450
7451                 if (params->action_is_for_table_entries)
7452                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7453                 if (params->action_is_for_default_entry)
7454                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7455
7456                 t->actions[i] = action_find(p, params->action_names[i]);
7457                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7458                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7459         }
7460         t->default_action = default_action;
7461         if (default_action->st)
7462                 memcpy(t->default_action_data,
7463                        params->default_action_data,
7464                        default_action->st->n_bits / 8);
7465         t->n_actions = params->n_actions;
7466         t->default_action_is_const = params->default_action_is_const;
7467         t->action_data_size_max = action_data_size_max;
7468
7469         t->size = size;
7470         t->id = p->n_tables;
7471
7472         /* Node add to tailq. */
7473         TAILQ_INSERT_TAIL(&p->tables, t, node);
7474         p->n_tables++;
7475
7476         return 0;
7477
7478 nomem:
7479         if (!t)
7480                 return -ENOMEM;
7481
7482         free(t->action_is_for_default_entry);
7483         free(t->action_is_for_table_entries);
7484         free(t->default_action_data);
7485         free(t->actions);
7486         free(t->fields);
7487         free(t);
7488
7489         return -ENOMEM;
7490 }
7491
7492 static struct rte_swx_table_params *
7493 table_params_get(struct table *table)
7494 {
7495         struct rte_swx_table_params *params;
7496         struct field *first, *last;
7497         uint8_t *key_mask;
7498         uint32_t key_size, key_offset, action_data_size, i;
7499
7500         /* Memory allocation. */
7501         params = calloc(1, sizeof(struct rte_swx_table_params));
7502         if (!params)
7503                 return NULL;
7504
7505         /* Find first (smallest offset) and last (biggest offset) match fields. */
7506         first = table->fields[0].field;
7507         last = table->fields[0].field;
7508
7509         for (i = 0; i < table->n_fields; i++) {
7510                 struct field *f = table->fields[i].field;
7511
7512                 if (f->offset < first->offset)
7513                         first = f;
7514
7515                 if (f->offset > last->offset)
7516                         last = f;
7517         }
7518
7519         /* Key offset and size. */
7520         key_offset = first->offset / 8;
7521         key_size = (last->offset + last->n_bits - first->offset) / 8;
7522
7523         /* Memory allocation. */
7524         key_mask = calloc(1, key_size);
7525         if (!key_mask) {
7526                 free(params);
7527                 return NULL;
7528         }
7529
7530         /* Key mask. */
7531         for (i = 0; i < table->n_fields; i++) {
7532                 struct field *f = table->fields[i].field;
7533                 uint32_t start = (f->offset - first->offset) / 8;
7534                 size_t size = f->n_bits / 8;
7535
7536                 memset(&key_mask[start], 0xFF, size);
7537         }
7538
7539         /* Action data size. */
7540         action_data_size = 0;
7541         for (i = 0; i < table->n_actions; i++) {
7542                 struct action *action = table->actions[i];
7543                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7544
7545                 if (ads > action_data_size)
7546                         action_data_size = ads;
7547         }
7548
7549         /* Fill in. */
7550         params->match_type = table->type->match_type;
7551         params->key_size = key_size;
7552         params->key_offset = key_offset;
7553         params->key_mask0 = key_mask;
7554         params->action_data_size = action_data_size;
7555         params->n_keys_max = table->size;
7556
7557         return params;
7558 }
7559
7560 static void
7561 table_params_free(struct rte_swx_table_params *params)
7562 {
7563         if (!params)
7564                 return;
7565
7566         free(params->key_mask0);
7567         free(params);
7568 }
7569
7570 static int
7571 table_stub_lkp(void *table __rte_unused,
7572                void *mailbox __rte_unused,
7573                uint8_t **key __rte_unused,
7574                uint64_t *action_id __rte_unused,
7575                uint8_t **action_data __rte_unused,
7576                int *hit)
7577 {
7578         *hit = 0;
7579         return 1; /* DONE. */
7580 }
7581
7582 static int
7583 table_build(struct rte_swx_pipeline *p)
7584 {
7585         uint32_t i;
7586
7587         /* Per pipeline: table statistics. */
7588         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7589         CHECK(p->table_stats, ENOMEM);
7590
7591         for (i = 0; i < p->n_tables; i++) {
7592                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7593                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7594         }
7595
7596         /* Per thread: table runt-time. */
7597         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7598                 struct thread *t = &p->threads[i];
7599                 struct table *table;
7600
7601                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7602                 CHECK(t->tables, ENOMEM);
7603
7604                 TAILQ_FOREACH(table, &p->tables, node) {
7605                         struct table_runtime *r = &t->tables[table->id];
7606
7607                         if (table->type) {
7608                                 uint64_t size;
7609
7610                                 size = table->type->ops.mailbox_size_get();
7611
7612                                 /* r->func. */
7613                                 r->func = table->type->ops.lkp;
7614
7615                                 /* r->mailbox. */
7616                                 if (size) {
7617                                         r->mailbox = calloc(1, size);
7618                                         CHECK(r->mailbox, ENOMEM);
7619                                 }
7620
7621                                 /* r->key. */
7622                                 r->key = table->header ?
7623                                         &t->structs[table->header->struct_id] :
7624                                         &t->structs[p->metadata_struct_id];
7625                         } else {
7626                                 r->func = table_stub_lkp;
7627                         }
7628                 }
7629         }
7630
7631         return 0;
7632 }
7633
7634 static void
7635 table_build_free(struct rte_swx_pipeline *p)
7636 {
7637         uint32_t i;
7638
7639         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7640                 struct thread *t = &p->threads[i];
7641                 uint32_t j;
7642
7643                 if (!t->tables)
7644                         continue;
7645
7646                 for (j = 0; j < p->n_tables; j++) {
7647                         struct table_runtime *r = &t->tables[j];
7648
7649                         free(r->mailbox);
7650                 }
7651
7652                 free(t->tables);
7653                 t->tables = NULL;
7654         }
7655
7656         if (p->table_stats) {
7657                 for (i = 0; i < p->n_tables; i++)
7658                         free(p->table_stats[i].n_pkts_action);
7659
7660                 free(p->table_stats);
7661         }
7662 }
7663
7664 static void
7665 table_free(struct rte_swx_pipeline *p)
7666 {
7667         table_build_free(p);
7668
7669         /* Tables. */
7670         for ( ; ; ) {
7671                 struct table *elem;
7672
7673                 elem = TAILQ_FIRST(&p->tables);
7674                 if (!elem)
7675                         break;
7676
7677                 TAILQ_REMOVE(&p->tables, elem, node);
7678                 free(elem->fields);
7679                 free(elem->actions);
7680                 free(elem->default_action_data);
7681                 free(elem);
7682         }
7683
7684         /* Table types. */
7685         for ( ; ; ) {
7686                 struct table_type *elem;
7687
7688                 elem = TAILQ_FIRST(&p->table_types);
7689                 if (!elem)
7690                         break;
7691
7692                 TAILQ_REMOVE(&p->table_types, elem, node);
7693                 free(elem);
7694         }
7695 }
7696
7697 /*
7698  * Selector.
7699  */
7700 static struct selector *
7701 selector_find(struct rte_swx_pipeline *p, const char *name)
7702 {
7703         struct selector *s;
7704
7705         TAILQ_FOREACH(s, &p->selectors, node)
7706                 if (strcmp(s->name, name) == 0)
7707                         return s;
7708
7709         return NULL;
7710 }
7711
7712 static struct selector *
7713 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7714 {
7715         struct selector *s = NULL;
7716
7717         TAILQ_FOREACH(s, &p->selectors, node)
7718                 if (s->id == id)
7719                         return s;
7720
7721         return NULL;
7722 }
7723
7724 static int
7725 selector_fields_check(struct rte_swx_pipeline *p,
7726                       struct rte_swx_pipeline_selector_params *params,
7727                       struct header **header)
7728 {
7729         struct header *h0 = NULL;
7730         struct field *hf, *mf;
7731         uint32_t i;
7732
7733         /* Return if no selector fields. */
7734         if (!params->n_selector_fields || !params->selector_field_names)
7735                 return -EINVAL;
7736
7737         /* Check that all the selector fields either belong to the same header
7738          * or are all meta-data fields.
7739          */
7740         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7741         mf = metadata_field_parse(p, params->selector_field_names[0]);
7742         if (!hf && !mf)
7743                 return -EINVAL;
7744
7745         for (i = 1; i < params->n_selector_fields; i++)
7746                 if (h0) {
7747                         struct header *h;
7748
7749                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7750                         if (!hf || (h->id != h0->id))
7751                                 return -EINVAL;
7752                 } else {
7753                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7754                         if (!mf)
7755                                 return -EINVAL;
7756                 }
7757
7758         /* Check that there are no duplicated match fields. */
7759         for (i = 0; i < params->n_selector_fields; i++) {
7760                 const char *field_name = params->selector_field_names[i];
7761                 uint32_t j;
7762
7763                 for (j = i + 1; j < params->n_selector_fields; j++)
7764                         if (!strcmp(params->selector_field_names[j], field_name))
7765                                 return -EINVAL;
7766         }
7767
7768         /* Return. */
7769         if (header)
7770                 *header = h0;
7771
7772         return 0;
7773 }
7774
7775 int
7776 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7777                                  const char *name,
7778                                  struct rte_swx_pipeline_selector_params *params)
7779 {
7780         struct selector *s;
7781         struct header *selector_header = NULL;
7782         struct field *group_id_field, *member_id_field;
7783         uint32_t i;
7784         int status = 0;
7785
7786         CHECK(p, EINVAL);
7787
7788         CHECK_NAME(name, EINVAL);
7789         CHECK(!table_find(p, name), EEXIST);
7790         CHECK(!selector_find(p, name), EEXIST);
7791         CHECK(!learner_find(p, name), EEXIST);
7792
7793         CHECK(params, EINVAL);
7794
7795         CHECK_NAME(params->group_id_field_name, EINVAL);
7796         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7797         CHECK(group_id_field, EINVAL);
7798
7799         for (i = 0; i < params->n_selector_fields; i++) {
7800                 const char *field_name = params->selector_field_names[i];
7801
7802                 CHECK_NAME(field_name, EINVAL);
7803         }
7804         status = selector_fields_check(p, params, &selector_header);
7805         if (status)
7806                 return status;
7807
7808         CHECK_NAME(params->member_id_field_name, EINVAL);
7809         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7810         CHECK(member_id_field, EINVAL);
7811
7812         CHECK(params->n_groups_max, EINVAL);
7813
7814         CHECK(params->n_members_per_group_max, EINVAL);
7815
7816         /* Memory allocation. */
7817         s = calloc(1, sizeof(struct selector));
7818         if (!s) {
7819                 status = -ENOMEM;
7820                 goto error;
7821         }
7822
7823         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7824         if (!s->selector_fields) {
7825                 status = -ENOMEM;
7826                 goto error;
7827         }
7828
7829         /* Node initialization. */
7830         strcpy(s->name, name);
7831
7832         s->group_id_field = group_id_field;
7833
7834         for (i = 0; i < params->n_selector_fields; i++) {
7835                 const char *field_name = params->selector_field_names[i];
7836
7837                 s->selector_fields[i] = selector_header ?
7838                         header_field_parse(p, field_name, NULL) :
7839                         metadata_field_parse(p, field_name);
7840         }
7841
7842         s->n_selector_fields = params->n_selector_fields;
7843
7844         s->selector_header = selector_header;
7845
7846         s->member_id_field = member_id_field;
7847
7848         s->n_groups_max = params->n_groups_max;
7849
7850         s->n_members_per_group_max = params->n_members_per_group_max;
7851
7852         s->id = p->n_selectors;
7853
7854         /* Node add to tailq. */
7855         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7856         p->n_selectors++;
7857
7858         return 0;
7859
7860 error:
7861         if (!s)
7862                 return status;
7863
7864         free(s->selector_fields);
7865
7866         free(s);
7867
7868         return status;
7869 }
7870
7871 static void
7872 selector_params_free(struct rte_swx_table_selector_params *params)
7873 {
7874         if (!params)
7875                 return;
7876
7877         free(params->selector_mask);
7878
7879         free(params);
7880 }
7881
7882 static struct rte_swx_table_selector_params *
7883 selector_table_params_get(struct selector *s)
7884 {
7885         struct rte_swx_table_selector_params *params = NULL;
7886         struct field *first, *last;
7887         uint32_t i;
7888
7889         /* Memory allocation. */
7890         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7891         if (!params)
7892                 goto error;
7893
7894         /* Group ID. */
7895         params->group_id_offset = s->group_id_field->offset / 8;
7896
7897         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7898         first = s->selector_fields[0];
7899         last = s->selector_fields[0];
7900
7901         for (i = 0; i < s->n_selector_fields; i++) {
7902                 struct field *f = s->selector_fields[i];
7903
7904                 if (f->offset < first->offset)
7905                         first = f;
7906
7907                 if (f->offset > last->offset)
7908                         last = f;
7909         }
7910
7911         /* Selector offset and size. */
7912         params->selector_offset = first->offset / 8;
7913         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7914
7915         /* Memory allocation. */
7916         params->selector_mask = calloc(1, params->selector_size);
7917         if (!params->selector_mask)
7918                 goto error;
7919
7920         /* Selector mask. */
7921         for (i = 0; i < s->n_selector_fields; i++) {
7922                 struct field *f = s->selector_fields[i];
7923                 uint32_t start = (f->offset - first->offset) / 8;
7924                 size_t size = f->n_bits / 8;
7925
7926                 memset(&params->selector_mask[start], 0xFF, size);
7927         }
7928
7929         /* Member ID. */
7930         params->member_id_offset = s->member_id_field->offset / 8;
7931
7932         /* Maximum number of groups. */
7933         params->n_groups_max = s->n_groups_max;
7934
7935         /* Maximum number of members per group. */
7936         params->n_members_per_group_max = s->n_members_per_group_max;
7937
7938         return params;
7939
7940 error:
7941         selector_params_free(params);
7942         return NULL;
7943 }
7944
7945 static void
7946 selector_build_free(struct rte_swx_pipeline *p)
7947 {
7948         uint32_t i;
7949
7950         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7951                 struct thread *t = &p->threads[i];
7952                 uint32_t j;
7953
7954                 if (!t->selectors)
7955                         continue;
7956
7957                 for (j = 0; j < p->n_selectors; j++) {
7958                         struct selector_runtime *r = &t->selectors[j];
7959
7960                         free(r->mailbox);
7961                 }
7962
7963                 free(t->selectors);
7964                 t->selectors = NULL;
7965         }
7966
7967         free(p->selector_stats);
7968         p->selector_stats = NULL;
7969 }
7970
7971 static int
7972 selector_build(struct rte_swx_pipeline *p)
7973 {
7974         uint32_t i;
7975         int status = 0;
7976
7977         /* Per pipeline: selector statistics. */
7978         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7979         if (!p->selector_stats) {
7980                 status = -ENOMEM;
7981                 goto error;
7982         }
7983
7984         /* Per thread: selector run-time. */
7985         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7986                 struct thread *t = &p->threads[i];
7987                 struct selector *s;
7988
7989                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7990                 if (!t->selectors) {
7991                         status = -ENOMEM;
7992                         goto error;
7993                 }
7994
7995                 TAILQ_FOREACH(s, &p->selectors, node) {
7996                         struct selector_runtime *r = &t->selectors[s->id];
7997                         uint64_t size;
7998
7999                         /* r->mailbox. */
8000                         size = rte_swx_table_selector_mailbox_size_get();
8001                         if (size) {
8002                                 r->mailbox = calloc(1, size);
8003                                 if (!r->mailbox) {
8004                                         status = -ENOMEM;
8005                                         goto error;
8006                                 }
8007                         }
8008
8009                         /* r->group_id_buffer. */
8010                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8011
8012                         /* r->selector_buffer. */
8013                         r->selector_buffer = s->selector_header ?
8014                                 &t->structs[s->selector_header->struct_id] :
8015                                 &t->structs[p->metadata_struct_id];
8016
8017                         /* r->member_id_buffer. */
8018                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8019                 }
8020         }
8021
8022         return 0;
8023
8024 error:
8025         selector_build_free(p);
8026         return status;
8027 }
8028
8029 static void
8030 selector_free(struct rte_swx_pipeline *p)
8031 {
8032         selector_build_free(p);
8033
8034         /* Selector tables. */
8035         for ( ; ; ) {
8036                 struct selector *elem;
8037
8038                 elem = TAILQ_FIRST(&p->selectors);
8039                 if (!elem)
8040                         break;
8041
8042                 TAILQ_REMOVE(&p->selectors, elem, node);
8043                 free(elem->selector_fields);
8044                 free(elem);
8045         }
8046 }
8047
8048 /*
8049  * Learner table.
8050  */
8051 static struct learner *
8052 learner_find(struct rte_swx_pipeline *p, const char *name)
8053 {
8054         struct learner *l;
8055
8056         TAILQ_FOREACH(l, &p->learners, node)
8057                 if (!strcmp(l->name, name))
8058                         return l;
8059
8060         return NULL;
8061 }
8062
8063 static struct learner *
8064 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8065 {
8066         struct learner *l = NULL;
8067
8068         TAILQ_FOREACH(l, &p->learners, node)
8069                 if (l->id == id)
8070                         return l;
8071
8072         return NULL;
8073 }
8074
8075 static int
8076 learner_match_fields_check(struct rte_swx_pipeline *p,
8077                            struct rte_swx_pipeline_learner_params *params,
8078                            struct header **header)
8079 {
8080         struct header *h0 = NULL;
8081         struct field *hf, *mf;
8082         uint32_t i;
8083
8084         /* Return if no match fields. */
8085         if (!params->n_fields || !params->field_names)
8086                 return -EINVAL;
8087
8088         /* Check that all the match fields either belong to the same header
8089          * or are all meta-data fields.
8090          */
8091         hf = header_field_parse(p, params->field_names[0], &h0);
8092         mf = metadata_field_parse(p, params->field_names[0]);
8093         if (!hf && !mf)
8094                 return -EINVAL;
8095
8096         for (i = 1; i < params->n_fields; i++)
8097                 if (h0) {
8098                         struct header *h;
8099
8100                         hf = header_field_parse(p, params->field_names[i], &h);
8101                         if (!hf || (h->id != h0->id))
8102                                 return -EINVAL;
8103                 } else {
8104                         mf = metadata_field_parse(p, params->field_names[i]);
8105                         if (!mf)
8106                                 return -EINVAL;
8107                 }
8108
8109         /* Check that there are no duplicated match fields. */
8110         for (i = 0; i < params->n_fields; i++) {
8111                 const char *field_name = params->field_names[i];
8112                 uint32_t j;
8113
8114                 for (j = i + 1; j < params->n_fields; j++)
8115                         if (!strcmp(params->field_names[j], field_name))
8116                                 return -EINVAL;
8117         }
8118
8119         /* Return. */
8120         if (header)
8121                 *header = h0;
8122
8123         return 0;
8124 }
8125
8126 static int
8127 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8128 {
8129         struct struct_type *mst = p->metadata_st, *ast = a->st;
8130         struct field *mf, *af;
8131         uint32_t mf_pos, i;
8132
8133         if (!ast) {
8134                 if (mf_name)
8135                         return -EINVAL;
8136
8137                 return 0;
8138         }
8139
8140         /* Check that mf_name is the name of a valid meta-data field. */
8141         CHECK_NAME(mf_name, EINVAL);
8142         mf = metadata_field_parse(p, mf_name);
8143         CHECK(mf, EINVAL);
8144
8145         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8146          * all the action arguments.
8147          */
8148         mf_pos = mf - mst->fields;
8149         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8150
8151         /* Check that the size of each of the identified meta-data fields matches exactly the size
8152          * of the corresponding action argument.
8153          */
8154         for (i = 0; i < ast->n_fields; i++) {
8155                 mf = &mst->fields[mf_pos + i];
8156                 af = &ast->fields[i];
8157
8158                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8159         }
8160
8161         return 0;
8162 }
8163
8164 static int
8165 learner_action_learning_check(struct rte_swx_pipeline *p,
8166                               struct action *action,
8167                               const char **action_names,
8168                               uint32_t n_actions)
8169 {
8170         uint32_t i;
8171
8172         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8173          * the action passed as argument to the "learn" instruction) is also enabled for the
8174          * current learner table.
8175          */
8176         for (i = 0; i < action->n_instructions; i++) {
8177                 struct instruction *instr = &action->instructions[i];
8178                 uint32_t found = 0, j;
8179
8180                 if (instr->type != INSTR_LEARNER_LEARN)
8181                         continue;
8182
8183                 for (j = 0; j < n_actions; j++) {
8184                         struct action *a;
8185
8186                         a = action_find(p, action_names[j]);
8187                         if (!a)
8188                                 return -EINVAL;
8189
8190                         if (a->id == instr->learn.action_id)
8191                                 found = 1;
8192                 }
8193
8194                 if (!found)
8195                         return -EINVAL;
8196         }
8197
8198         return 0;
8199 }
8200
8201 int
8202 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8203                               const char *name,
8204                               struct rte_swx_pipeline_learner_params *params,
8205                               uint32_t size,
8206                               uint32_t timeout)
8207 {
8208         struct learner *l = NULL;
8209         struct action *default_action;
8210         struct header *header = NULL;
8211         uint32_t action_data_size_max = 0, i;
8212         int status = 0;
8213
8214         CHECK(p, EINVAL);
8215
8216         CHECK_NAME(name, EINVAL);
8217         CHECK(!table_find(p, name), EEXIST);
8218         CHECK(!selector_find(p, name), EEXIST);
8219         CHECK(!learner_find(p, name), EEXIST);
8220
8221         CHECK(params, EINVAL);
8222
8223         /* Match checks. */
8224         status = learner_match_fields_check(p, params, &header);
8225         if (status)
8226                 return status;
8227
8228         /* Action checks. */
8229         CHECK(params->n_actions, EINVAL);
8230         CHECK(params->action_names, EINVAL);
8231         for (i = 0; i < params->n_actions; i++) {
8232                 const char *action_name = params->action_names[i];
8233                 struct action *a;
8234                 uint32_t action_data_size;
8235                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8236
8237                 CHECK_NAME(action_name, EINVAL);
8238
8239                 a = action_find(p, action_name);
8240                 CHECK(a, EINVAL);
8241
8242                 status = learner_action_learning_check(p,
8243                                                        a,
8244                                                        params->action_names,
8245                                                        params->n_actions);
8246                 if (status)
8247                         return status;
8248
8249                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8250                 if (action_data_size > action_data_size_max)
8251                         action_data_size_max = action_data_size;
8252
8253                 if (params->action_is_for_table_entries)
8254                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8255                 if (params->action_is_for_default_entry)
8256                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8257                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8258         }
8259
8260         CHECK_NAME(params->default_action_name, EINVAL);
8261         for (i = 0; i < p->n_actions; i++)
8262                 if (!strcmp(params->action_names[i],
8263                             params->default_action_name))
8264                         break;
8265         CHECK(i < params->n_actions, EINVAL);
8266         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8267               EINVAL);
8268
8269         default_action = action_find(p, params->default_action_name);
8270         CHECK((default_action->st && params->default_action_data) ||
8271               !params->default_action_data, EINVAL);
8272
8273         /* Any other checks. */
8274         CHECK(size, EINVAL);
8275         CHECK(timeout, EINVAL);
8276
8277         /* Memory allocation. */
8278         l = calloc(1, sizeof(struct learner));
8279         if (!l)
8280                 goto nomem;
8281
8282         l->fields = calloc(params->n_fields, sizeof(struct field *));
8283         if (!l->fields)
8284                 goto nomem;
8285
8286         l->actions = calloc(params->n_actions, sizeof(struct action *));
8287         if (!l->actions)
8288                 goto nomem;
8289
8290         if (action_data_size_max) {
8291                 l->default_action_data = calloc(1, action_data_size_max);
8292                 if (!l->default_action_data)
8293                         goto nomem;
8294         }
8295
8296         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8297         if (!l->action_is_for_table_entries)
8298                 goto nomem;
8299
8300         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8301         if (!l->action_is_for_default_entry)
8302                 goto nomem;
8303
8304         /* Node initialization. */
8305         strcpy(l->name, name);
8306
8307         for (i = 0; i < params->n_fields; i++) {
8308                 const char *field_name = params->field_names[i];
8309
8310                 l->fields[i] = header ?
8311                         header_field_parse(p, field_name, NULL) :
8312                         metadata_field_parse(p, field_name);
8313         }
8314
8315         l->n_fields = params->n_fields;
8316
8317         l->header = header;
8318
8319         for (i = 0; i < params->n_actions; i++) {
8320                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8321
8322                 if (params->action_is_for_table_entries)
8323                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8324                 if (params->action_is_for_default_entry)
8325                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8326
8327                 l->actions[i] = action_find(p, params->action_names[i]);
8328                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8329                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8330         }
8331
8332         l->default_action = default_action;
8333
8334         if (default_action->st)
8335                 memcpy(l->default_action_data,
8336                        params->default_action_data,
8337                        default_action->st->n_bits / 8);
8338
8339         l->n_actions = params->n_actions;
8340
8341         l->default_action_is_const = params->default_action_is_const;
8342
8343         l->action_data_size_max = action_data_size_max;
8344
8345         l->size = size;
8346
8347         l->timeout = timeout;
8348
8349         l->id = p->n_learners;
8350
8351         /* Node add to tailq. */
8352         TAILQ_INSERT_TAIL(&p->learners, l, node);
8353         p->n_learners++;
8354
8355         return 0;
8356
8357 nomem:
8358         if (!l)
8359                 return -ENOMEM;
8360
8361         free(l->action_is_for_default_entry);
8362         free(l->action_is_for_table_entries);
8363         free(l->default_action_data);
8364         free(l->actions);
8365         free(l->fields);
8366         free(l);
8367
8368         return -ENOMEM;
8369 }
8370
8371 static void
8372 learner_params_free(struct rte_swx_table_learner_params *params)
8373 {
8374         if (!params)
8375                 return;
8376
8377         free(params->key_mask0);
8378
8379         free(params);
8380 }
8381
8382 static struct rte_swx_table_learner_params *
8383 learner_params_get(struct learner *l)
8384 {
8385         struct rte_swx_table_learner_params *params = NULL;
8386         struct field *first, *last;
8387         uint32_t i;
8388
8389         /* Memory allocation. */
8390         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8391         if (!params)
8392                 goto error;
8393
8394         /* Find first (smallest offset) and last (biggest offset) match fields. */
8395         first = l->fields[0];
8396         last = l->fields[0];
8397
8398         for (i = 0; i < l->n_fields; i++) {
8399                 struct field *f = l->fields[i];
8400
8401                 if (f->offset < first->offset)
8402                         first = f;
8403
8404                 if (f->offset > last->offset)
8405                         last = f;
8406         }
8407
8408         /* Key offset and size. */
8409         params->key_offset = first->offset / 8;
8410         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8411
8412         /* Memory allocation. */
8413         params->key_mask0 = calloc(1, params->key_size);
8414         if (!params->key_mask0)
8415                 goto error;
8416
8417         /* Key mask. */
8418         for (i = 0; i < l->n_fields; i++) {
8419                 struct field *f = l->fields[i];
8420                 uint32_t start = (f->offset - first->offset) / 8;
8421                 size_t size = f->n_bits / 8;
8422
8423                 memset(&params->key_mask0[start], 0xFF, size);
8424         }
8425
8426         /* Action data size. */
8427         params->action_data_size = l->action_data_size_max;
8428
8429         /* Maximum number of keys. */
8430         params->n_keys_max = l->size;
8431
8432         /* Timeout. */
8433         params->key_timeout = l->timeout;
8434
8435         return params;
8436
8437 error:
8438         learner_params_free(params);
8439         return NULL;
8440 }
8441
8442 static void
8443 learner_build_free(struct rte_swx_pipeline *p)
8444 {
8445         uint32_t i;
8446
8447         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8448                 struct thread *t = &p->threads[i];
8449                 uint32_t j;
8450
8451                 if (!t->learners)
8452                         continue;
8453
8454                 for (j = 0; j < p->n_learners; j++) {
8455                         struct learner_runtime *r = &t->learners[j];
8456
8457                         free(r->mailbox);
8458                 }
8459
8460                 free(t->learners);
8461                 t->learners = NULL;
8462         }
8463
8464         if (p->learner_stats) {
8465                 for (i = 0; i < p->n_learners; i++)
8466                         free(p->learner_stats[i].n_pkts_action);
8467
8468                 free(p->learner_stats);
8469         }
8470 }
8471
8472 static int
8473 learner_build(struct rte_swx_pipeline *p)
8474 {
8475         uint32_t i;
8476         int status = 0;
8477
8478         /* Per pipeline: learner statistics. */
8479         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8480         CHECK(p->learner_stats, ENOMEM);
8481
8482         for (i = 0; i < p->n_learners; i++) {
8483                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8484                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8485         }
8486
8487         /* Per thread: learner run-time. */
8488         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8489                 struct thread *t = &p->threads[i];
8490                 struct learner *l;
8491
8492                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8493                 if (!t->learners) {
8494                         status = -ENOMEM;
8495                         goto error;
8496                 }
8497
8498                 TAILQ_FOREACH(l, &p->learners, node) {
8499                         struct learner_runtime *r = &t->learners[l->id];
8500                         uint64_t size;
8501
8502                         /* r->mailbox. */
8503                         size = rte_swx_table_learner_mailbox_size_get();
8504                         if (size) {
8505                                 r->mailbox = calloc(1, size);
8506                                 if (!r->mailbox) {
8507                                         status = -ENOMEM;
8508                                         goto error;
8509                                 }
8510                         }
8511
8512                         /* r->key. */
8513                         r->key = l->header ?
8514                                 &t->structs[l->header->struct_id] :
8515                                 &t->structs[p->metadata_struct_id];
8516                 }
8517         }
8518
8519         return 0;
8520
8521 error:
8522         learner_build_free(p);
8523         return status;
8524 }
8525
8526 static void
8527 learner_free(struct rte_swx_pipeline *p)
8528 {
8529         learner_build_free(p);
8530
8531         /* Learner tables. */
8532         for ( ; ; ) {
8533                 struct learner *l;
8534
8535                 l = TAILQ_FIRST(&p->learners);
8536                 if (!l)
8537                         break;
8538
8539                 TAILQ_REMOVE(&p->learners, l, node);
8540                 free(l->fields);
8541                 free(l->actions);
8542                 free(l->default_action_data);
8543                 free(l);
8544         }
8545 }
8546
8547 /*
8548  * Table state.
8549  */
8550 static int
8551 table_state_build(struct rte_swx_pipeline *p)
8552 {
8553         struct table *table;
8554         struct selector *s;
8555         struct learner *l;
8556
8557         p->table_state = calloc(p->n_tables + p->n_selectors,
8558                                 sizeof(struct rte_swx_table_state));
8559         CHECK(p->table_state, ENOMEM);
8560
8561         TAILQ_FOREACH(table, &p->tables, node) {
8562                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8563
8564                 if (table->type) {
8565                         struct rte_swx_table_params *params;
8566
8567                         /* ts->obj. */
8568                         params = table_params_get(table);
8569                         CHECK(params, ENOMEM);
8570
8571                         ts->obj = table->type->ops.create(params,
8572                                 NULL,
8573                                 table->args,
8574                                 p->numa_node);
8575
8576                         table_params_free(params);
8577                         CHECK(ts->obj, ENODEV);
8578                 }
8579
8580                 /* ts->default_action_data. */
8581                 if (table->action_data_size_max) {
8582                         ts->default_action_data =
8583                                 malloc(table->action_data_size_max);
8584                         CHECK(ts->default_action_data, ENOMEM);
8585
8586                         memcpy(ts->default_action_data,
8587                                table->default_action_data,
8588                                table->action_data_size_max);
8589                 }
8590
8591                 /* ts->default_action_id. */
8592                 ts->default_action_id = table->default_action->id;
8593         }
8594
8595         TAILQ_FOREACH(s, &p->selectors, node) {
8596                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8597                 struct rte_swx_table_selector_params *params;
8598
8599                 /* ts->obj. */
8600                 params = selector_table_params_get(s);
8601                 CHECK(params, ENOMEM);
8602
8603                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8604
8605                 selector_params_free(params);
8606                 CHECK(ts->obj, ENODEV);
8607         }
8608
8609         TAILQ_FOREACH(l, &p->learners, node) {
8610                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8611                         p->n_selectors + l->id];
8612                 struct rte_swx_table_learner_params *params;
8613
8614                 /* ts->obj. */
8615                 params = learner_params_get(l);
8616                 CHECK(params, ENOMEM);
8617
8618                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8619                 learner_params_free(params);
8620                 CHECK(ts->obj, ENODEV);
8621
8622                 /* ts->default_action_data. */
8623                 if (l->action_data_size_max) {
8624                         ts->default_action_data = malloc(l->action_data_size_max);
8625                         CHECK(ts->default_action_data, ENOMEM);
8626
8627                         memcpy(ts->default_action_data,
8628                                l->default_action_data,
8629                                l->action_data_size_max);
8630                 }
8631
8632                 /* ts->default_action_id. */
8633                 ts->default_action_id = l->default_action->id;
8634         }
8635
8636         return 0;
8637 }
8638
8639 static void
8640 table_state_build_free(struct rte_swx_pipeline *p)
8641 {
8642         uint32_t i;
8643
8644         if (!p->table_state)
8645                 return;
8646
8647         for (i = 0; i < p->n_tables; i++) {
8648                 struct rte_swx_table_state *ts = &p->table_state[i];
8649                 struct table *table = table_find_by_id(p, i);
8650
8651                 /* ts->obj. */
8652                 if (table->type && ts->obj)
8653                         table->type->ops.free(ts->obj);
8654
8655                 /* ts->default_action_data. */
8656                 free(ts->default_action_data);
8657         }
8658
8659         for (i = 0; i < p->n_selectors; i++) {
8660                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8661
8662                 /* ts->obj. */
8663                 if (ts->obj)
8664                         rte_swx_table_selector_free(ts->obj);
8665         }
8666
8667         for (i = 0; i < p->n_learners; i++) {
8668                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8669
8670                 /* ts->obj. */
8671                 if (ts->obj)
8672                         rte_swx_table_learner_free(ts->obj);
8673
8674                 /* ts->default_action_data. */
8675                 free(ts->default_action_data);
8676         }
8677
8678         free(p->table_state);
8679         p->table_state = NULL;
8680 }
8681
8682 static void
8683 table_state_free(struct rte_swx_pipeline *p)
8684 {
8685         table_state_build_free(p);
8686 }
8687
8688 /*
8689  * Register array.
8690  */
8691 static struct regarray *
8692 regarray_find(struct rte_swx_pipeline *p, const char *name)
8693 {
8694         struct regarray *elem;
8695
8696         TAILQ_FOREACH(elem, &p->regarrays, node)
8697                 if (!strcmp(elem->name, name))
8698                         return elem;
8699
8700         return NULL;
8701 }
8702
8703 static struct regarray *
8704 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8705 {
8706         struct regarray *elem = NULL;
8707
8708         TAILQ_FOREACH(elem, &p->regarrays, node)
8709                 if (elem->id == id)
8710                         return elem;
8711
8712         return NULL;
8713 }
8714
8715 int
8716 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8717                               const char *name,
8718                               uint32_t size,
8719                               uint64_t init_val)
8720 {
8721         struct regarray *r;
8722
8723         CHECK(p, EINVAL);
8724
8725         CHECK_NAME(name, EINVAL);
8726         CHECK(!regarray_find(p, name), EEXIST);
8727
8728         CHECK(size, EINVAL);
8729         size = rte_align32pow2(size);
8730
8731         /* Memory allocation. */
8732         r = calloc(1, sizeof(struct regarray));
8733         CHECK(r, ENOMEM);
8734
8735         /* Node initialization. */
8736         strcpy(r->name, name);
8737         r->init_val = init_val;
8738         r->size = size;
8739         r->id = p->n_regarrays;
8740
8741         /* Node add to tailq. */
8742         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8743         p->n_regarrays++;
8744
8745         return 0;
8746 }
8747
8748 static int
8749 regarray_build(struct rte_swx_pipeline *p)
8750 {
8751         struct regarray *regarray;
8752
8753         if (!p->n_regarrays)
8754                 return 0;
8755
8756         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8757         CHECK(p->regarray_runtime, ENOMEM);
8758
8759         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8760                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8761                 uint32_t i;
8762
8763                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8764                                          RTE_CACHE_LINE_SIZE,
8765                                          p->numa_node);
8766                 CHECK(r->regarray, ENOMEM);
8767
8768                 if (regarray->init_val)
8769                         for (i = 0; i < regarray->size; i++)
8770                                 r->regarray[i] = regarray->init_val;
8771
8772                 r->size_mask = regarray->size - 1;
8773         }
8774
8775         return 0;
8776 }
8777
8778 static void
8779 regarray_build_free(struct rte_swx_pipeline *p)
8780 {
8781         uint32_t i;
8782
8783         if (!p->regarray_runtime)
8784                 return;
8785
8786         for (i = 0; i < p->n_regarrays; i++) {
8787                 struct regarray *regarray = regarray_find_by_id(p, i);
8788                 struct regarray_runtime *r = &p->regarray_runtime[i];
8789
8790                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8791         }
8792
8793         free(p->regarray_runtime);
8794         p->regarray_runtime = NULL;
8795 }
8796
8797 static void
8798 regarray_free(struct rte_swx_pipeline *p)
8799 {
8800         regarray_build_free(p);
8801
8802         for ( ; ; ) {
8803                 struct regarray *elem;
8804
8805                 elem = TAILQ_FIRST(&p->regarrays);
8806                 if (!elem)
8807                         break;
8808
8809                 TAILQ_REMOVE(&p->regarrays, elem, node);
8810                 free(elem);
8811         }
8812 }
8813
8814 /*
8815  * Meter array.
8816  */
8817 static struct meter_profile *
8818 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8819 {
8820         struct meter_profile *elem;
8821
8822         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8823                 if (!strcmp(elem->name, name))
8824                         return elem;
8825
8826         return NULL;
8827 }
8828
8829 static struct metarray *
8830 metarray_find(struct rte_swx_pipeline *p, const char *name)
8831 {
8832         struct metarray *elem;
8833
8834         TAILQ_FOREACH(elem, &p->metarrays, node)
8835                 if (!strcmp(elem->name, name))
8836                         return elem;
8837
8838         return NULL;
8839 }
8840
8841 static struct metarray *
8842 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8843 {
8844         struct metarray *elem = NULL;
8845
8846         TAILQ_FOREACH(elem, &p->metarrays, node)
8847                 if (elem->id == id)
8848                         return elem;
8849
8850         return NULL;
8851 }
8852
8853 int
8854 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8855                                  const char *name,
8856                                  uint32_t size)
8857 {
8858         struct metarray *m;
8859
8860         CHECK(p, EINVAL);
8861
8862         CHECK_NAME(name, EINVAL);
8863         CHECK(!metarray_find(p, name), EEXIST);
8864
8865         CHECK(size, EINVAL);
8866         size = rte_align32pow2(size);
8867
8868         /* Memory allocation. */
8869         m = calloc(1, sizeof(struct metarray));
8870         CHECK(m, ENOMEM);
8871
8872         /* Node initialization. */
8873         strcpy(m->name, name);
8874         m->size = size;
8875         m->id = p->n_metarrays;
8876
8877         /* Node add to tailq. */
8878         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8879         p->n_metarrays++;
8880
8881         return 0;
8882 }
8883
8884 struct meter_profile meter_profile_default = {
8885         .node = {0},
8886         .name = "",
8887         .params = {0},
8888
8889         .profile = {
8890                 .cbs = 10000,
8891                 .pbs = 10000,
8892                 .cir_period = 1,
8893                 .cir_bytes_per_period = 1,
8894                 .pir_period = 1,
8895                 .pir_bytes_per_period = 1,
8896         },
8897
8898         .n_users = 0,
8899 };
8900
8901 static void
8902 meter_init(struct meter *m)
8903 {
8904         memset(m, 0, sizeof(struct meter));
8905         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8906         m->profile = &meter_profile_default;
8907         m->color_mask = RTE_COLOR_GREEN;
8908
8909         meter_profile_default.n_users++;
8910 }
8911
8912 static int
8913 metarray_build(struct rte_swx_pipeline *p)
8914 {
8915         struct metarray *m;
8916
8917         if (!p->n_metarrays)
8918                 return 0;
8919
8920         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8921         CHECK(p->metarray_runtime, ENOMEM);
8922
8923         TAILQ_FOREACH(m, &p->metarrays, node) {
8924                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8925                 uint32_t i;
8926
8927                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8928                                          RTE_CACHE_LINE_SIZE,
8929                                          p->numa_node);
8930                 CHECK(r->metarray, ENOMEM);
8931
8932                 for (i = 0; i < m->size; i++)
8933                         meter_init(&r->metarray[i]);
8934
8935                 r->size_mask = m->size - 1;
8936         }
8937
8938         return 0;
8939 }
8940
8941 static void
8942 metarray_build_free(struct rte_swx_pipeline *p)
8943 {
8944         uint32_t i;
8945
8946         if (!p->metarray_runtime)
8947                 return;
8948
8949         for (i = 0; i < p->n_metarrays; i++) {
8950                 struct metarray *m = metarray_find_by_id(p, i);
8951                 struct metarray_runtime *r = &p->metarray_runtime[i];
8952
8953                 env_free(r->metarray, m->size * sizeof(struct meter));
8954         }
8955
8956         free(p->metarray_runtime);
8957         p->metarray_runtime = NULL;
8958 }
8959
8960 static void
8961 metarray_free(struct rte_swx_pipeline *p)
8962 {
8963         metarray_build_free(p);
8964
8965         /* Meter arrays. */
8966         for ( ; ; ) {
8967                 struct metarray *elem;
8968
8969                 elem = TAILQ_FIRST(&p->metarrays);
8970                 if (!elem)
8971                         break;
8972
8973                 TAILQ_REMOVE(&p->metarrays, elem, node);
8974                 free(elem);
8975         }
8976
8977         /* Meter profiles. */
8978         for ( ; ; ) {
8979                 struct meter_profile *elem;
8980
8981                 elem = TAILQ_FIRST(&p->meter_profiles);
8982                 if (!elem)
8983                         break;
8984
8985                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8986                 free(elem);
8987         }
8988 }
8989
8990 /*
8991  * Pipeline.
8992  */
8993 void
8994 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8995 {
8996         void *lib;
8997
8998         if (!p)
8999                 return;
9000
9001         lib = p->lib;
9002
9003         free(p->instruction_data);
9004         free(p->instructions);
9005
9006         metarray_free(p);
9007         regarray_free(p);
9008         table_state_free(p);
9009         learner_free(p);
9010         selector_free(p);
9011         table_free(p);
9012         action_free(p);
9013         instruction_table_free(p);
9014         metadata_free(p);
9015         header_free(p);
9016         extern_func_free(p);
9017         extern_obj_free(p);
9018         port_out_free(p);
9019         port_in_free(p);
9020         struct_free(p);
9021
9022         free(p);
9023
9024         if (lib)
9025                 dlclose(lib);
9026 }
9027
9028 static int
9029 port_in_types_register(struct rte_swx_pipeline *p)
9030 {
9031         int status;
9032
9033         status = rte_swx_pipeline_port_in_type_register(p,
9034                 "ethdev",
9035                 &rte_swx_port_ethdev_reader_ops);
9036         if (status)
9037                 return status;
9038
9039         status = rte_swx_pipeline_port_in_type_register(p,
9040                 "ring",
9041                 &rte_swx_port_ring_reader_ops);
9042         if (status)
9043                 return status;
9044
9045 #ifdef RTE_PORT_PCAP
9046         status = rte_swx_pipeline_port_in_type_register(p,
9047                 "source",
9048                 &rte_swx_port_source_ops);
9049         if (status)
9050                 return status;
9051 #endif
9052
9053         status = rte_swx_pipeline_port_in_type_register(p,
9054                 "fd",
9055                 &rte_swx_port_fd_reader_ops);
9056         if (status)
9057                 return status;
9058
9059         return 0;
9060 }
9061
9062 static int
9063 port_out_types_register(struct rte_swx_pipeline *p)
9064 {
9065         int status;
9066
9067         status = rte_swx_pipeline_port_out_type_register(p,
9068                 "ethdev",
9069                 &rte_swx_port_ethdev_writer_ops);
9070         if (status)
9071                 return status;
9072
9073         status = rte_swx_pipeline_port_out_type_register(p,
9074                 "ring",
9075                 &rte_swx_port_ring_writer_ops);
9076         if (status)
9077                 return status;
9078
9079         status = rte_swx_pipeline_port_out_type_register(p,
9080                 "sink",
9081                 &rte_swx_port_sink_ops);
9082         if (status)
9083                 return status;
9084
9085         status = rte_swx_pipeline_port_out_type_register(p,
9086                 "fd",
9087                 &rte_swx_port_fd_writer_ops);
9088         if (status)
9089                 return status;
9090
9091         return 0;
9092 }
9093
9094 static int
9095 table_types_register(struct rte_swx_pipeline *p)
9096 {
9097         int status;
9098
9099         status = rte_swx_pipeline_table_type_register(p,
9100                 "exact",
9101                 RTE_SWX_TABLE_MATCH_EXACT,
9102                 &rte_swx_table_exact_match_ops);
9103         if (status)
9104                 return status;
9105
9106         status = rte_swx_pipeline_table_type_register(p,
9107                 "wildcard",
9108                 RTE_SWX_TABLE_MATCH_WILDCARD,
9109                 &rte_swx_table_wildcard_match_ops);
9110         if (status)
9111                 return status;
9112
9113         return 0;
9114 }
9115
9116 int
9117 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9118 {
9119         struct rte_swx_pipeline *pipeline = NULL;
9120         int status = 0;
9121
9122         /* Check input parameters. */
9123         CHECK(p, EINVAL);
9124
9125         /* Memory allocation. */
9126         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9127         if (!pipeline) {
9128                 status = -ENOMEM;
9129                 goto error;
9130         }
9131
9132         /* Initialization. */
9133         TAILQ_INIT(&pipeline->struct_types);
9134         TAILQ_INIT(&pipeline->port_in_types);
9135         TAILQ_INIT(&pipeline->ports_in);
9136         TAILQ_INIT(&pipeline->port_out_types);
9137         TAILQ_INIT(&pipeline->ports_out);
9138         TAILQ_INIT(&pipeline->extern_types);
9139         TAILQ_INIT(&pipeline->extern_objs);
9140         TAILQ_INIT(&pipeline->extern_funcs);
9141         TAILQ_INIT(&pipeline->headers);
9142         TAILQ_INIT(&pipeline->actions);
9143         TAILQ_INIT(&pipeline->table_types);
9144         TAILQ_INIT(&pipeline->tables);
9145         TAILQ_INIT(&pipeline->selectors);
9146         TAILQ_INIT(&pipeline->learners);
9147         TAILQ_INIT(&pipeline->regarrays);
9148         TAILQ_INIT(&pipeline->meter_profiles);
9149         TAILQ_INIT(&pipeline->metarrays);
9150
9151         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9152         pipeline->numa_node = numa_node;
9153
9154         status = port_in_types_register(pipeline);
9155         if (status)
9156                 goto error;
9157
9158         status = port_out_types_register(pipeline);
9159         if (status)
9160                 goto error;
9161
9162         status = table_types_register(pipeline);
9163         if (status)
9164                 goto error;
9165
9166         *p = pipeline;
9167         return 0;
9168
9169 error:
9170         rte_swx_pipeline_free(pipeline);
9171         return status;
9172 }
9173
9174 int
9175 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9176                                      const char **instructions,
9177                                      uint32_t n_instructions)
9178 {
9179         int err;
9180         uint32_t i;
9181
9182         err = instruction_config(p, NULL, instructions, n_instructions);
9183         if (err)
9184                 return err;
9185
9186         /* Thread instruction pointer reset. */
9187         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9188                 struct thread *t = &p->threads[i];
9189
9190                 thread_ip_reset(p, t);
9191         }
9192
9193         return 0;
9194 }
9195
9196 static int
9197 pipeline_compile(struct rte_swx_pipeline *p);
9198
9199 int
9200 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9201 {
9202         int status;
9203
9204         CHECK(p, EINVAL);
9205         CHECK(p->build_done == 0, EEXIST);
9206
9207         status = port_in_build(p);
9208         if (status)
9209                 goto error;
9210
9211         status = port_out_build(p);
9212         if (status)
9213                 goto error;
9214
9215         status = struct_build(p);
9216         if (status)
9217                 goto error;
9218
9219         status = extern_obj_build(p);
9220         if (status)
9221                 goto error;
9222
9223         status = extern_func_build(p);
9224         if (status)
9225                 goto error;
9226
9227         status = header_build(p);
9228         if (status)
9229                 goto error;
9230
9231         status = metadata_build(p);
9232         if (status)
9233                 goto error;
9234
9235         status = instruction_table_build(p);
9236         if (status)
9237                 goto error;
9238
9239         status = action_build(p);
9240         if (status)
9241                 goto error;
9242
9243         status = table_build(p);
9244         if (status)
9245                 goto error;
9246
9247         status = selector_build(p);
9248         if (status)
9249                 goto error;
9250
9251         status = learner_build(p);
9252         if (status)
9253                 goto error;
9254
9255         status = table_state_build(p);
9256         if (status)
9257                 goto error;
9258
9259         status = regarray_build(p);
9260         if (status)
9261                 goto error;
9262
9263         status = metarray_build(p);
9264         if (status)
9265                 goto error;
9266
9267         p->build_done = 1;
9268
9269         pipeline_compile(p);
9270
9271         return 0;
9272
9273 error:
9274         metarray_build_free(p);
9275         regarray_build_free(p);
9276         table_state_build_free(p);
9277         learner_build_free(p);
9278         selector_build_free(p);
9279         table_build_free(p);
9280         action_build_free(p);
9281         instruction_table_build_free(p);
9282         metadata_build_free(p);
9283         header_build_free(p);
9284         extern_func_build_free(p);
9285         extern_obj_build_free(p);
9286         port_out_build_free(p);
9287         port_in_build_free(p);
9288         struct_build_free(p);
9289
9290         return status;
9291 }
9292
9293 void
9294 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9295 {
9296         uint32_t i;
9297
9298         for (i = 0; i < n_instructions; i++)
9299                 instr_exec(p);
9300 }
9301
9302 void
9303 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9304 {
9305         uint32_t i;
9306
9307         for (i = 0; i < p->n_ports_out; i++) {
9308                 struct port_out_runtime *port = &p->out[i];
9309
9310                 if (port->flush)
9311                         port->flush(port->obj);
9312         }
9313 }
9314
9315 /*
9316  * Control.
9317  */
9318 int
9319 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9320                               struct rte_swx_ctl_pipeline_info *pipeline)
9321 {
9322         struct action *action;
9323         struct table *table;
9324         uint32_t n_actions = 0, n_tables = 0;
9325
9326         if (!p || !pipeline)
9327                 return -EINVAL;
9328
9329         TAILQ_FOREACH(action, &p->actions, node)
9330                 n_actions++;
9331
9332         TAILQ_FOREACH(table, &p->tables, node)
9333                 n_tables++;
9334
9335         pipeline->n_ports_in = p->n_ports_in;
9336         pipeline->n_ports_out = p->n_ports_out;
9337         pipeline->n_actions = n_actions;
9338         pipeline->n_tables = n_tables;
9339         pipeline->n_selectors = p->n_selectors;
9340         pipeline->n_learners = p->n_learners;
9341         pipeline->n_regarrays = p->n_regarrays;
9342         pipeline->n_metarrays = p->n_metarrays;
9343
9344         return 0;
9345 }
9346
9347 int
9348 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9349 {
9350         if (!p || !numa_node)
9351                 return -EINVAL;
9352
9353         *numa_node = p->numa_node;
9354         return 0;
9355 }
9356
9357 int
9358 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9359                             uint32_t action_id,
9360                             struct rte_swx_ctl_action_info *action)
9361 {
9362         struct action *a = NULL;
9363
9364         if (!p || (action_id >= p->n_actions) || !action)
9365                 return -EINVAL;
9366
9367         a = action_find_by_id(p, action_id);
9368         if (!a)
9369                 return -EINVAL;
9370
9371         strcpy(action->name, a->name);
9372         action->n_args = a->st ? a->st->n_fields : 0;
9373         return 0;
9374 }
9375
9376 int
9377 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9378                                 uint32_t action_id,
9379                                 uint32_t action_arg_id,
9380                                 struct rte_swx_ctl_action_arg_info *action_arg)
9381 {
9382         struct action *a = NULL;
9383         struct field *arg = NULL;
9384
9385         if (!p || (action_id >= p->n_actions) || !action_arg)
9386                 return -EINVAL;
9387
9388         a = action_find_by_id(p, action_id);
9389         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9390                 return -EINVAL;
9391
9392         arg = &a->st->fields[action_arg_id];
9393         strcpy(action_arg->name, arg->name);
9394         action_arg->n_bits = arg->n_bits;
9395         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9396
9397         return 0;
9398 }
9399
9400 int
9401 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9402                            uint32_t table_id,
9403                            struct rte_swx_ctl_table_info *table)
9404 {
9405         struct table *t = NULL;
9406
9407         if (!p || !table)
9408                 return -EINVAL;
9409
9410         t = table_find_by_id(p, table_id);
9411         if (!t)
9412                 return -EINVAL;
9413
9414         strcpy(table->name, t->name);
9415         strcpy(table->args, t->args);
9416         table->n_match_fields = t->n_fields;
9417         table->n_actions = t->n_actions;
9418         table->default_action_is_const = t->default_action_is_const;
9419         table->size = t->size;
9420         return 0;
9421 }
9422
9423 int
9424 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9425         uint32_t table_id,
9426         uint32_t match_field_id,
9427         struct rte_swx_ctl_table_match_field_info *match_field)
9428 {
9429         struct table *t;
9430         struct match_field *f;
9431
9432         if (!p || (table_id >= p->n_tables) || !match_field)
9433                 return -EINVAL;
9434
9435         t = table_find_by_id(p, table_id);
9436         if (!t || (match_field_id >= t->n_fields))
9437                 return -EINVAL;
9438
9439         f = &t->fields[match_field_id];
9440         match_field->match_type = f->match_type;
9441         match_field->is_header = t->header ? 1 : 0;
9442         match_field->n_bits = f->field->n_bits;
9443         match_field->offset = f->field->offset;
9444
9445         return 0;
9446 }
9447
9448 int
9449 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9450         uint32_t table_id,
9451         uint32_t table_action_id,
9452         struct rte_swx_ctl_table_action_info *table_action)
9453 {
9454         struct table *t;
9455
9456         if (!p || (table_id >= p->n_tables) || !table_action)
9457                 return -EINVAL;
9458
9459         t = table_find_by_id(p, table_id);
9460         if (!t || (table_action_id >= t->n_actions))
9461                 return -EINVAL;
9462
9463         table_action->action_id = t->actions[table_action_id]->id;
9464
9465         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9466         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9467
9468         return 0;
9469 }
9470
9471 int
9472 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9473                           uint32_t table_id,
9474                           struct rte_swx_table_ops *table_ops,
9475                           int *is_stub)
9476 {
9477         struct table *t;
9478
9479         if (!p || (table_id >= p->n_tables))
9480                 return -EINVAL;
9481
9482         t = table_find_by_id(p, table_id);
9483         if (!t)
9484                 return -EINVAL;
9485
9486         if (t->type) {
9487                 if (table_ops)
9488                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9489                 *is_stub = 0;
9490         } else {
9491                 *is_stub = 1;
9492         }
9493
9494         return 0;
9495 }
9496
9497 int
9498 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9499                               uint32_t selector_id,
9500                               struct rte_swx_ctl_selector_info *selector)
9501 {
9502         struct selector *s = NULL;
9503
9504         if (!p || !selector)
9505                 return -EINVAL;
9506
9507         s = selector_find_by_id(p, selector_id);
9508         if (!s)
9509                 return -EINVAL;
9510
9511         strcpy(selector->name, s->name);
9512
9513         selector->n_selector_fields = s->n_selector_fields;
9514         selector->n_groups_max = s->n_groups_max;
9515         selector->n_members_per_group_max = s->n_members_per_group_max;
9516
9517         return 0;
9518 }
9519
9520 int
9521 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9522          uint32_t selector_id,
9523          struct rte_swx_ctl_table_match_field_info *field)
9524 {
9525         struct selector *s;
9526
9527         if (!p || (selector_id >= p->n_selectors) || !field)
9528                 return -EINVAL;
9529
9530         s = selector_find_by_id(p, selector_id);
9531         if (!s)
9532                 return -EINVAL;
9533
9534         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9535         field->is_header = 0;
9536         field->n_bits = s->group_id_field->n_bits;
9537         field->offset = s->group_id_field->offset;
9538
9539         return 0;
9540 }
9541
9542 int
9543 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9544          uint32_t selector_id,
9545          uint32_t selector_field_id,
9546          struct rte_swx_ctl_table_match_field_info *field)
9547 {
9548         struct selector *s;
9549         struct field *f;
9550
9551         if (!p || (selector_id >= p->n_selectors) || !field)
9552                 return -EINVAL;
9553
9554         s = selector_find_by_id(p, selector_id);
9555         if (!s || (selector_field_id >= s->n_selector_fields))
9556                 return -EINVAL;
9557
9558         f = s->selector_fields[selector_field_id];
9559         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9560         field->is_header = s->selector_header ? 1 : 0;
9561         field->n_bits = f->n_bits;
9562         field->offset = f->offset;
9563
9564         return 0;
9565 }
9566
9567 int
9568 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9569          uint32_t selector_id,
9570          struct rte_swx_ctl_table_match_field_info *field)
9571 {
9572         struct selector *s;
9573
9574         if (!p || (selector_id >= p->n_selectors) || !field)
9575                 return -EINVAL;
9576
9577         s = selector_find_by_id(p, selector_id);
9578         if (!s)
9579                 return -EINVAL;
9580
9581         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9582         field->is_header = 0;
9583         field->n_bits = s->member_id_field->n_bits;
9584         field->offset = s->member_id_field->offset;
9585
9586         return 0;
9587 }
9588
9589 int
9590 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9591                              uint32_t learner_id,
9592                              struct rte_swx_ctl_learner_info *learner)
9593 {
9594         struct learner *l = NULL;
9595
9596         if (!p || !learner)
9597                 return -EINVAL;
9598
9599         l = learner_find_by_id(p, learner_id);
9600         if (!l)
9601                 return -EINVAL;
9602
9603         strcpy(learner->name, l->name);
9604
9605         learner->n_match_fields = l->n_fields;
9606         learner->n_actions = l->n_actions;
9607         learner->default_action_is_const = l->default_action_is_const;
9608         learner->size = l->size;
9609
9610         return 0;
9611 }
9612
9613 int
9614 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9615                                          uint32_t learner_id,
9616                                          uint32_t match_field_id,
9617                                          struct rte_swx_ctl_table_match_field_info *match_field)
9618 {
9619         struct learner *l;
9620         struct field *f;
9621
9622         if (!p || (learner_id >= p->n_learners) || !match_field)
9623                 return -EINVAL;
9624
9625         l = learner_find_by_id(p, learner_id);
9626         if (!l || (match_field_id >= l->n_fields))
9627                 return -EINVAL;
9628
9629         f = l->fields[match_field_id];
9630         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9631         match_field->is_header = l->header ? 1 : 0;
9632         match_field->n_bits = f->n_bits;
9633         match_field->offset = f->offset;
9634
9635         return 0;
9636 }
9637
9638 int
9639 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9640                                     uint32_t learner_id,
9641                                     uint32_t learner_action_id,
9642                                     struct rte_swx_ctl_table_action_info *learner_action)
9643 {
9644         struct learner *l;
9645
9646         if (!p || (learner_id >= p->n_learners) || !learner_action)
9647                 return -EINVAL;
9648
9649         l = learner_find_by_id(p, learner_id);
9650         if (!l || (learner_action_id >= l->n_actions))
9651                 return -EINVAL;
9652
9653         learner_action->action_id = l->actions[learner_action_id]->id;
9654
9655         learner_action->action_is_for_table_entries =
9656                 l->action_is_for_table_entries[learner_action_id];
9657
9658         learner_action->action_is_for_default_entry =
9659                 l->action_is_for_default_entry[learner_action_id];
9660
9661         return 0;
9662 }
9663
9664 int
9665 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9666                                  struct rte_swx_table_state **table_state)
9667 {
9668         if (!p || !table_state || !p->build_done)
9669                 return -EINVAL;
9670
9671         *table_state = p->table_state;
9672         return 0;
9673 }
9674
9675 int
9676 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9677                                  struct rte_swx_table_state *table_state)
9678 {
9679         if (!p || !table_state || !p->build_done)
9680                 return -EINVAL;
9681
9682         p->table_state = table_state;
9683         return 0;
9684 }
9685
9686 int
9687 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9688                                         uint32_t port_id,
9689                                         struct rte_swx_port_in_stats *stats)
9690 {
9691         struct port_in *port;
9692
9693         if (!p || !stats)
9694                 return -EINVAL;
9695
9696         port = port_in_find(p, port_id);
9697         if (!port)
9698                 return -EINVAL;
9699
9700         port->type->ops.stats_read(port->obj, stats);
9701         return 0;
9702 }
9703
9704 int
9705 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9706                                          uint32_t port_id,
9707                                          struct rte_swx_port_out_stats *stats)
9708 {
9709         struct port_out *port;
9710
9711         if (!p || !stats)
9712                 return -EINVAL;
9713
9714         port = port_out_find(p, port_id);
9715         if (!port)
9716                 return -EINVAL;
9717
9718         port->type->ops.stats_read(port->obj, stats);
9719         return 0;
9720 }
9721
9722 int
9723 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9724                                       const char *table_name,
9725                                       struct rte_swx_table_stats *stats)
9726 {
9727         struct table *table;
9728         struct table_statistics *table_stats;
9729
9730         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9731                 return -EINVAL;
9732
9733         table = table_find(p, table_name);
9734         if (!table)
9735                 return -EINVAL;
9736
9737         table_stats = &p->table_stats[table->id];
9738
9739         memcpy(stats->n_pkts_action,
9740                table_stats->n_pkts_action,
9741                p->n_actions * sizeof(uint64_t));
9742
9743         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9744         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9745
9746         return 0;
9747 }
9748
9749 int
9750 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9751         const char *selector_name,
9752         struct rte_swx_pipeline_selector_stats *stats)
9753 {
9754         struct selector *s;
9755
9756         if (!p || !selector_name || !selector_name[0] || !stats)
9757                 return -EINVAL;
9758
9759         s = selector_find(p, selector_name);
9760         if (!s)
9761                 return -EINVAL;
9762
9763         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9764
9765         return 0;
9766 }
9767
9768 int
9769 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9770                                         const char *learner_name,
9771                                         struct rte_swx_learner_stats *stats)
9772 {
9773         struct learner *l;
9774         struct learner_statistics *learner_stats;
9775
9776         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9777                 return -EINVAL;
9778
9779         l = learner_find(p, learner_name);
9780         if (!l)
9781                 return -EINVAL;
9782
9783         learner_stats = &p->learner_stats[l->id];
9784
9785         memcpy(stats->n_pkts_action,
9786                learner_stats->n_pkts_action,
9787                p->n_actions * sizeof(uint64_t));
9788
9789         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9790         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9791
9792         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9793         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9794
9795         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9796
9797         return 0;
9798 }
9799
9800 int
9801 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9802                               uint32_t regarray_id,
9803                               struct rte_swx_ctl_regarray_info *regarray)
9804 {
9805         struct regarray *r;
9806
9807         if (!p || !regarray)
9808                 return -EINVAL;
9809
9810         r = regarray_find_by_id(p, regarray_id);
9811         if (!r)
9812                 return -EINVAL;
9813
9814         strcpy(regarray->name, r->name);
9815         regarray->size = r->size;
9816         return 0;
9817 }
9818
9819 int
9820 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9821                                    const char *regarray_name,
9822                                    uint32_t regarray_index,
9823                                    uint64_t *value)
9824 {
9825         struct regarray *regarray;
9826         struct regarray_runtime *r;
9827
9828         if (!p || !regarray_name || !value)
9829                 return -EINVAL;
9830
9831         regarray = regarray_find(p, regarray_name);
9832         if (!regarray || (regarray_index >= regarray->size))
9833                 return -EINVAL;
9834
9835         r = &p->regarray_runtime[regarray->id];
9836         *value = r->regarray[regarray_index];
9837         return 0;
9838 }
9839
9840 int
9841 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9842                                    const char *regarray_name,
9843                                    uint32_t regarray_index,
9844                                    uint64_t value)
9845 {
9846         struct regarray *regarray;
9847         struct regarray_runtime *r;
9848
9849         if (!p || !regarray_name)
9850                 return -EINVAL;
9851
9852         regarray = regarray_find(p, regarray_name);
9853         if (!regarray || (regarray_index >= regarray->size))
9854                 return -EINVAL;
9855
9856         r = &p->regarray_runtime[regarray->id];
9857         r->regarray[regarray_index] = value;
9858         return 0;
9859 }
9860
9861 int
9862 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9863                               uint32_t metarray_id,
9864                               struct rte_swx_ctl_metarray_info *metarray)
9865 {
9866         struct metarray *m;
9867
9868         if (!p || !metarray)
9869                 return -EINVAL;
9870
9871         m = metarray_find_by_id(p, metarray_id);
9872         if (!m)
9873                 return -EINVAL;
9874
9875         strcpy(metarray->name, m->name);
9876         metarray->size = m->size;
9877         return 0;
9878 }
9879
9880 int
9881 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9882                               const char *name,
9883                               struct rte_meter_trtcm_params *params)
9884 {
9885         struct meter_profile *mp;
9886         int status;
9887
9888         CHECK(p, EINVAL);
9889         CHECK_NAME(name, EINVAL);
9890         CHECK(params, EINVAL);
9891         CHECK(!meter_profile_find(p, name), EEXIST);
9892
9893         /* Node allocation. */
9894         mp = calloc(1, sizeof(struct meter_profile));
9895         CHECK(mp, ENOMEM);
9896
9897         /* Node initialization. */
9898         strcpy(mp->name, name);
9899         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9900         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9901         if (status) {
9902                 free(mp);
9903                 CHECK(0, EINVAL);
9904         }
9905
9906         /* Node add to tailq. */
9907         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9908
9909         return 0;
9910 }
9911
9912 int
9913 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9914                                  const char *name)
9915 {
9916         struct meter_profile *mp;
9917
9918         CHECK(p, EINVAL);
9919         CHECK_NAME(name, EINVAL);
9920
9921         mp = meter_profile_find(p, name);
9922         CHECK(mp, EINVAL);
9923         CHECK(!mp->n_users, EBUSY);
9924
9925         /* Remove node from tailq. */
9926         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9927         free(mp);
9928
9929         return 0;
9930 }
9931
9932 int
9933 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9934                         const char *metarray_name,
9935                         uint32_t metarray_index)
9936 {
9937         struct meter_profile *mp_old;
9938         struct metarray *metarray;
9939         struct metarray_runtime *metarray_runtime;
9940         struct meter *m;
9941
9942         CHECK(p, EINVAL);
9943         CHECK_NAME(metarray_name, EINVAL);
9944
9945         metarray = metarray_find(p, metarray_name);
9946         CHECK(metarray, EINVAL);
9947         CHECK(metarray_index < metarray->size, EINVAL);
9948
9949         metarray_runtime = &p->metarray_runtime[metarray->id];
9950         m = &metarray_runtime->metarray[metarray_index];
9951         mp_old = m->profile;
9952
9953         meter_init(m);
9954
9955         mp_old->n_users--;
9956
9957         return 0;
9958 }
9959
9960 int
9961 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9962                       const char *metarray_name,
9963                       uint32_t metarray_index,
9964                       const char *profile_name)
9965 {
9966         struct meter_profile *mp, *mp_old;
9967         struct metarray *metarray;
9968         struct metarray_runtime *metarray_runtime;
9969         struct meter *m;
9970
9971         CHECK(p, EINVAL);
9972         CHECK_NAME(metarray_name, EINVAL);
9973
9974         metarray = metarray_find(p, metarray_name);
9975         CHECK(metarray, EINVAL);
9976         CHECK(metarray_index < metarray->size, EINVAL);
9977
9978         mp = meter_profile_find(p, profile_name);
9979         CHECK(mp, EINVAL);
9980
9981         metarray_runtime = &p->metarray_runtime[metarray->id];
9982         m = &metarray_runtime->metarray[metarray_index];
9983         mp_old = m->profile;
9984
9985         memset(m, 0, sizeof(struct meter));
9986         rte_meter_trtcm_config(&m->m, &mp->profile);
9987         m->profile = mp;
9988         m->color_mask = RTE_COLORS;
9989
9990         mp->n_users++;
9991         mp_old->n_users--;
9992
9993         return 0;
9994 }
9995
9996 int
9997 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9998                              const char *metarray_name,
9999                              uint32_t metarray_index,
10000                              struct rte_swx_ctl_meter_stats *stats)
10001 {
10002         struct metarray *metarray;
10003         struct metarray_runtime *metarray_runtime;
10004         struct meter *m;
10005
10006         CHECK(p, EINVAL);
10007         CHECK_NAME(metarray_name, EINVAL);
10008
10009         metarray = metarray_find(p, metarray_name);
10010         CHECK(metarray, EINVAL);
10011         CHECK(metarray_index < metarray->size, EINVAL);
10012
10013         CHECK(stats, EINVAL);
10014
10015         metarray_runtime = &p->metarray_runtime[metarray->id];
10016         m = &metarray_runtime->metarray[metarray_index];
10017
10018         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10019         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10020
10021         return 0;
10022 }
10023
10024 /*
10025  * Pipeline compilation.
10026  */
10027 static const char *
10028 instr_type_to_name(struct instruction *instr)
10029 {
10030         switch (instr->type) {
10031         case INSTR_RX: return "INSTR_RX";
10032
10033         case INSTR_TX: return "INSTR_TX";
10034         case INSTR_TX_I: return "INSTR_TX_I";
10035         case INSTR_DROP: return "INSTR_DROP";
10036
10037         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10038         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10039         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10040         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10041         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10042         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10043         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10044         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10045
10046         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10047
10048         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10049
10050         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10051         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10052         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10053         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10054         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10055         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10056         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10057         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10058         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10059
10060         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10061         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10062
10063         case INSTR_MOV: return "INSTR_MOV";
10064         case INSTR_MOV_MH: return "INSTR_MOV_MH";
10065         case INSTR_MOV_HM: return "INSTR_MOV_HM";
10066         case INSTR_MOV_HH: return "INSTR_MOV_HH";
10067         case INSTR_MOV_I: return "INSTR_MOV_I";
10068
10069         case INSTR_DMA_HT: return "INSTR_DMA_HT";
10070         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10071         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10072         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10073         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10074         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10075         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10076         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10077
10078         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10079         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10080         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10081         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10082         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10083         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10084
10085         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10086         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10087         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10088         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10089         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10090         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10091
10092         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10093         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10094         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10095         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10096
10097         case INSTR_ALU_AND: return "INSTR_ALU_AND";
10098         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10099         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10100         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10101         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10102
10103         case INSTR_ALU_OR: return "INSTR_ALU_OR";
10104         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10105         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10106         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10107         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10108
10109         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10110         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10111         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10112         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10113         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10114
10115         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10116         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10117         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10118         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10119         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10120         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10121
10122         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10123         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10124         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10125         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10126         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10127         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10128
10129         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10130         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10131         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10132
10133         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10134         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10135         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10136         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10137         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10138         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10139
10140         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10141         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10142         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10143         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10144         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10145         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10146         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10147         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10148         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10149
10150         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10151         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10152         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10153         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10154         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10155         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10156         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10157         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10158         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10159
10160         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10161         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10162         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10163
10164         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10165         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10166         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10167         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10168         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10169         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10170         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10171         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10172         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10173         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10174         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10175         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10176
10177         case INSTR_TABLE: return "INSTR_TABLE";
10178         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10179         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10180         case INSTR_LEARNER: return "INSTR_LEARNER";
10181         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10182
10183         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10184         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10185
10186         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10187         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10188
10189         case INSTR_JMP: return "INSTR_JMP";
10190         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10191         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10192         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10193         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10194         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10195         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10196         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10197         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10198         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10199         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10200         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10201         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10202         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10203         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10204         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10205         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10206         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10207         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10208         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10209         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10210         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10211         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10212         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10213         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10214         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10215         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10216         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10217         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10218
10219         case INSTR_RETURN: return "INSTR_RETURN";
10220
10221         default: return "INSTR_UNKNOWN";
10222         }
10223 }
10224
10225 typedef void
10226 (*instruction_export_t)(struct instruction *, FILE *);
10227
10228 static void
10229 instr_io_export(struct instruction *instr, FILE *f)
10230 {
10231         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10232
10233         /* n_io, n_io_imm, n_hdrs. */
10234         if (instr->type == INSTR_RX ||
10235             instr->type == INSTR_TX ||
10236             instr->type == INSTR_HDR_EXTRACT_M ||
10237             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10238                 n_io = 1;
10239
10240         if (instr->type == INSTR_TX_I)
10241                 n_io_imm = 1;
10242
10243         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10244                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10245
10246         if (instr->type == INSTR_HDR_EXTRACT_M ||
10247             instr->type == INSTR_HDR_LOOKAHEAD ||
10248             instr->type == INSTR_HDR_EMIT)
10249                 n_hdrs = 1;
10250
10251         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10252                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10253
10254         /* instr. */
10255         fprintf(f,
10256                 "\t{\n"
10257                 "\t\t.type = %s,\n",
10258                 instr_type_to_name(instr));
10259
10260         /* instr.io. */
10261         if (n_io || n_io_imm || n_hdrs)
10262                 fprintf(f,
10263                         "\t\t.io = {\n");
10264
10265         /* instr.io.io. */
10266         if (n_io)
10267                 fprintf(f,
10268                         "\t\t\t.io = {\n"
10269                         "\t\t\t\t.offset = %u,\n"
10270                         "\t\t\t\t.n_bits = %u,\n"
10271                         "\t\t\t},\n",
10272                         instr->io.io.offset,
10273                         instr->io.io.n_bits);
10274
10275         if (n_io_imm)
10276                 fprintf(f,
10277                         "\t\t\t.io = {\n"
10278                         "\t\t\t\t.val = %u,\n"
10279                         "\t\t\t},\n",
10280                         instr->io.io.val);
10281
10282         /* instr.io.hdr. */
10283         if (n_hdrs) {
10284                 fprintf(f,
10285                         "\t\t.hdr = {\n");
10286
10287                 /* instr.io.hdr.header_id. */
10288                 fprintf(f,
10289                         "\t\t\t.header_id = {");
10290
10291                 for (i = 0; i < n_hdrs; i++)
10292                         fprintf(f,
10293                                 "%u, ",
10294                                 instr->io.hdr.header_id[i]);
10295
10296                 fprintf(f,
10297                         "},\n");
10298
10299                 /* instr.io.hdr.struct_id. */
10300                 fprintf(f,
10301                         "\t\t\t.struct_id = {");
10302
10303                 for (i = 0; i < n_hdrs; i++)
10304                         fprintf(f,
10305                                 "%u, ",
10306                                 instr->io.hdr.struct_id[i]);
10307
10308                 fprintf(f,
10309                         "},\n");
10310
10311                 /* instr.io.hdr.n_bytes. */
10312                 fprintf(f,
10313                         "\t\t\t.n_bytes = {");
10314
10315                 for (i = 0; i < n_hdrs; i++)
10316                         fprintf(f,
10317                                 "%u, ",
10318                                 instr->io.hdr.n_bytes[i]);
10319
10320                 fprintf(f,
10321                         "},\n");
10322
10323                 /* instr.io.hdr - closing curly brace. */
10324                 fprintf(f,
10325                         "\t\t\t}\n,");
10326         }
10327
10328         /* instr.io - closing curly brace. */
10329         if (n_io || n_io_imm || n_hdrs)
10330                 fprintf(f,
10331                         "\t\t},\n");
10332
10333         /* instr - closing curly brace. */
10334         fprintf(f,
10335                 "\t},\n");
10336 }
10337
10338 static void
10339 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10340 {
10341         fprintf(f,
10342                 "\t{\n"
10343                 "\t\t.type = %s,\n"
10344                 "\t\t.valid = {\n"
10345                 "\t\t\t.header_id = %u,\n"
10346                 "\t\t},\n"
10347                 "\t},\n",
10348                 instr_type_to_name(instr),
10349                 instr->valid.header_id);
10350 }
10351
10352 static void
10353 instr_mov_export(struct instruction *instr, FILE *f)
10354 {
10355         if (instr->type != INSTR_MOV_I)
10356                 fprintf(f,
10357                         "\t{\n"
10358                         "\t\t.type = %s,\n"
10359                         "\t\t.mov = {\n"
10360                         "\t\t\t.dst = {\n"
10361                         "\t\t\t\t.struct_id = %u,\n"
10362                         "\t\t\t\t.n_bits = %u,\n"
10363                         "\t\t\t\t.offset = %u,\n"
10364                         "\t\t\t},\n"
10365                         "\t\t\t.src = {\n"
10366                         "\t\t\t\t.struct_id = %u,\n"
10367                         "\t\t\t\t.n_bits = %u,\n"
10368                         "\t\t\t\t.offset = %u,\n"
10369                         "\t\t\t},\n"
10370                         "\t\t},\n"
10371                         "\t},\n",
10372                         instr_type_to_name(instr),
10373                         instr->mov.dst.struct_id,
10374                         instr->mov.dst.n_bits,
10375                         instr->mov.dst.offset,
10376                         instr->mov.src.struct_id,
10377                         instr->mov.src.n_bits,
10378                         instr->mov.src.offset);
10379         else
10380                 fprintf(f,
10381                         "\t{\n"
10382                         "\t\t.type = %s,\n"
10383                         "\t\t.mov = {\n"
10384                         "\t\t\t.dst = {\n"
10385                         "\t\t\t\t.struct_id = %u,\n"
10386                         "\t\t\t\t.n_bits = %u,\n"
10387                         "\t\t\t\t.offset = %u,\n"
10388                         "\t\t\t}\n,"
10389                         "\t\t\t.src_val = %" PRIu64 ",\n"
10390                         "\t\t},\n"
10391                         "\t},\n",
10392                         instr_type_to_name(instr),
10393                         instr->mov.dst.struct_id,
10394                         instr->mov.dst.n_bits,
10395                         instr->mov.dst.offset,
10396                         instr->mov.src_val);
10397 }
10398
10399 static void
10400 instr_dma_ht_export(struct instruction *instr, FILE *f)
10401 {
10402         uint32_t n_dma = 0, i;
10403
10404         /* n_dma. */
10405         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10406
10407         /* instr. */
10408         fprintf(f,
10409                 "\t{\n"
10410                 "\t\t.type = %s,\n",
10411                 instr_type_to_name(instr));
10412
10413         /* instr.dma. */
10414         fprintf(f,
10415                 "\t\t.dma = {\n");
10416
10417         /* instr.dma.dst. */
10418         fprintf(f,
10419                 "\t\t\t.dst = {\n");
10420
10421         /* instr.dma.dst.header_id. */
10422         fprintf(f,
10423                 "\t\t\t\t.header_id = {");
10424
10425         for (i = 0; i < n_dma; i++)
10426                 fprintf(f,
10427                         "%u, ",
10428                         instr->dma.dst.header_id[i]);
10429
10430         fprintf(f,
10431                 "},\n");
10432
10433         /* instr.dma.dst.struct_id. */
10434         fprintf(f,
10435                 "\t\t\t\t.struct_id = {");
10436
10437         for (i = 0; i < n_dma; i++)
10438                 fprintf(f,
10439                         "%u, ",
10440                         instr->dma.dst.struct_id[i]);
10441
10442         fprintf(f,
10443                 "},\n");
10444
10445         /* instr.dma.dst - closing curly brace. */
10446         fprintf(f,
10447                 "\t\t\t},\n");
10448
10449         /* instr.dma.src. */
10450         fprintf(f,
10451                 "\t\t\t.src = {\n");
10452
10453         /* instr.dma.src.offset. */
10454         fprintf(f,
10455                 "\t\t\t\t.offset = {");
10456
10457         for (i = 0; i < n_dma; i++)
10458                 fprintf(f,
10459                         "%u, ",
10460                         instr->dma.src.offset[i]);
10461
10462         fprintf(f,
10463                 "},\n");
10464
10465         /* instr.dma.src - closing curly brace. */
10466         fprintf(f,
10467                 "\t\t\t},\n");
10468
10469         /* instr.dma.n_bytes. */
10470         fprintf(f,
10471                 "\t\t\t.n_bytes = {");
10472
10473         for (i = 0; i < n_dma; i++)
10474                 fprintf(f,
10475                         "%u, ",
10476                         instr->dma.n_bytes[i]);
10477
10478         fprintf(f,
10479                 "},\n");
10480
10481         /* instr.dma - closing curly brace. */
10482         fprintf(f,
10483                 "\t\t},\n");
10484
10485         /* instr - closing curly brace. */
10486         fprintf(f,
10487                 "\t},\n");
10488 }
10489
10490 static void
10491 instr_alu_export(struct instruction *instr, FILE *f)
10492 {
10493         int imm = 0;
10494
10495         if (instr->type == INSTR_ALU_ADD_MI ||
10496             instr->type == INSTR_ALU_ADD_HI ||
10497             instr->type == INSTR_ALU_SUB_MI ||
10498             instr->type == INSTR_ALU_SUB_HI ||
10499             instr->type == INSTR_ALU_SHL_MI ||
10500             instr->type == INSTR_ALU_SHL_HI ||
10501             instr->type == INSTR_ALU_SHR_MI ||
10502             instr->type == INSTR_ALU_SHR_HI ||
10503             instr->type == INSTR_ALU_AND_I ||
10504             instr->type == INSTR_ALU_OR_I ||
10505             instr->type == INSTR_ALU_XOR_I)
10506                 imm = 1;
10507
10508         if (!imm)
10509                 fprintf(f,
10510                         "\t{\n"
10511                         "\t\t.type = %s,\n"
10512                         "\t\t.alu = {\n"
10513                         "\t\t\t.dst = {\n"
10514                         "\t\t\t\t.struct_id = %u,\n"
10515                         "\t\t\t\t.n_bits = %u,\n"
10516                         "\t\t\t\t.offset = %u,\n"
10517                         "\t\t\t},\n"
10518                         "\t\t\t.src = {\n"
10519                         "\t\t\t\t.struct_id = %u,\n"
10520                         "\t\t\t\t.n_bits = %u,\n"
10521                         "\t\t\t\t.offset = %u,\n"
10522                         "\t\t\t},\n"
10523                         "\t\t},\n"
10524                         "\t},\n",
10525                         instr_type_to_name(instr),
10526                         instr->alu.dst.struct_id,
10527                         instr->alu.dst.n_bits,
10528                         instr->alu.dst.offset,
10529                         instr->alu.src.struct_id,
10530                         instr->alu.src.n_bits,
10531                         instr->alu.src.offset);
10532         else
10533                 fprintf(f,
10534                         "\t{\n"
10535                         "\t\t.type = %s,\n"
10536                         "\t\t.alu = {\n"
10537                         "\t\t\t.dst = {\n"
10538                         "\t\t\t\t.struct_id = %u,\n"
10539                         "\t\t\t\t.n_bits = %u,\n"
10540                         "\t\t\t\t.offset = %u,\n"
10541                         "\t\t\t}\n,"
10542                         "\t\t\t.src_val = %" PRIu64 ",\n"
10543                         "\t\t},\n"
10544                         "\t},\n",
10545                         instr_type_to_name(instr),
10546                         instr->alu.dst.struct_id,
10547                         instr->alu.dst.n_bits,
10548                         instr->alu.dst.offset,
10549                         instr->alu.src_val);
10550 }
10551
10552 static void
10553 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10554 {
10555         int prefetch  = 0, idx_imm = 0, src_imm = 0;
10556
10557         if (instr->type == INSTR_REGPREFETCH_RH ||
10558             instr->type == INSTR_REGPREFETCH_RM ||
10559             instr->type == INSTR_REGPREFETCH_RI)
10560                 prefetch = 1;
10561
10562         /* index is the 3rd operand for the regrd instruction and the 2nd
10563          * operand for the regwr and regadd instructions.
10564          */
10565         if (instr->type == INSTR_REGPREFETCH_RI ||
10566             instr->type == INSTR_REGRD_HRI ||
10567             instr->type == INSTR_REGRD_MRI ||
10568             instr->type == INSTR_REGWR_RIH ||
10569             instr->type == INSTR_REGWR_RIM ||
10570             instr->type == INSTR_REGWR_RII ||
10571             instr->type == INSTR_REGADD_RIH ||
10572             instr->type == INSTR_REGADD_RIM ||
10573             instr->type == INSTR_REGADD_RII)
10574                 idx_imm = 1;
10575
10576         /* src is the 3rd operand for the regwr and regadd instructions. */
10577         if (instr->type == INSTR_REGWR_RHI ||
10578             instr->type == INSTR_REGWR_RMI ||
10579             instr->type == INSTR_REGWR_RII ||
10580             instr->type == INSTR_REGADD_RHI ||
10581             instr->type == INSTR_REGADD_RMI ||
10582             instr->type == INSTR_REGADD_RII)
10583                 src_imm = 1;
10584
10585         /* instr.regarray.regarray_id. */
10586         fprintf(f,
10587                 "\t{\n"
10588                 "\t\t.type = %s,\n"
10589                 "\t\t.regarray = {\n"
10590                 "\t\t\t.regarray_id = %u,\n",
10591                 instr_type_to_name(instr),
10592                 instr->regarray.regarray_id);
10593
10594         /* instr.regarray.idx / instr.regarray.idx_val. */
10595         if (!idx_imm)
10596                 fprintf(f,
10597                         "\t\t\t\t.idx = {\n"
10598                         "\t\t\t\t\t.struct_id = %u,\n"
10599                         "\t\t\t\t\t.n_bits = %u,\n"
10600                         "\t\t\t\t\t.offset = %u,\n"
10601                         "\t\t\t\t},\n",
10602                         instr->regarray.idx.struct_id,
10603                         instr->regarray.idx.n_bits,
10604                         instr->regarray.idx.offset);
10605         else
10606                 fprintf(f,
10607                         "\t\t\t\t.idx_val = %u,\n",
10608                         instr->regarray.idx_val);
10609
10610         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10611         if (!prefetch) {
10612                 if (!src_imm)
10613                         fprintf(f,
10614                                 "\t\t\t\t.dstsrc = {\n"
10615                                 "\t\t\t\t\t.struct_id = %u,\n"
10616                                 "\t\t\t\t\t.n_bits = %u,\n"
10617                                 "\t\t\t\t\t.offset = %u,\n"
10618                                 "\t\t\t\t},\n",
10619                                 instr->regarray.dstsrc.struct_id,
10620                                 instr->regarray.dstsrc.n_bits,
10621                                 instr->regarray.dstsrc.offset);
10622                 else
10623                         fprintf(f,
10624                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10625                                 instr->regarray.dstsrc_val);
10626         }
10627
10628         /* instr.regarray and instr - closing curly braces. */
10629         fprintf(f,
10630                 "\t\t},\n"
10631                 "\t},\n");
10632 }
10633
10634 static void
10635 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10636 {
10637         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
10638
10639         if (instr->type == INSTR_METPREFETCH_H ||
10640             instr->type == INSTR_METPREFETCH_M ||
10641             instr->type == INSTR_METPREFETCH_I)
10642                 prefetch = 1;
10643
10644         /* idx_imm. */
10645         if (instr->type == INSTR_METPREFETCH_I ||
10646             instr->type == INSTR_METER_IHM ||
10647             instr->type == INSTR_METER_IHI ||
10648             instr->type == INSTR_METER_IMM ||
10649             instr->type == INSTR_METER_IMI)
10650                 idx_imm = 1;
10651
10652         /* color_in_imm. */
10653         if (instr->type == INSTR_METER_HHI ||
10654             instr->type == INSTR_METER_HMI ||
10655             instr->type == INSTR_METER_MHI ||
10656             instr->type == INSTR_METER_MMI ||
10657             instr->type == INSTR_METER_IHI ||
10658             instr->type == INSTR_METER_IMI)
10659                 color_in_imm = 1;
10660
10661         /* instr.meter.metarray_id. */
10662         fprintf(f,
10663                 "\t{\n"
10664                 "\t\t.type = %s,\n"
10665                 "\t\t.meter = {\n"
10666                 "\t\t\t.metarray_id = %u,\n",
10667                 instr_type_to_name(instr),
10668                 instr->meter.metarray_id);
10669
10670         /* instr.meter.idx / instr.meter.idx_val. */
10671         if (!idx_imm)
10672                 fprintf(f,
10673                         "\t\t\t.idx = {\n"
10674                         "\t\t\t\t.struct_id = %u,\n"
10675                         "\t\t\t\t.n_bits = %u,\n"
10676                         "\t\t\t\t.offset = %u,\n"
10677                         "\t\t\t},\n",
10678                         instr->meter.idx.struct_id,
10679                         instr->meter.idx.n_bits,
10680                         instr->meter.idx.offset);
10681         else
10682                 fprintf(f,
10683                         "\t\t\t.idx_val = %u,\n",
10684                         instr->meter.idx_val);
10685
10686         if (!prefetch) {
10687                 /* instr.meter.length. */
10688                 fprintf(f,
10689                         "\t\t\t.length = {\n"
10690                         "\t\t\t\t.struct_id = %u,\n"
10691                         "\t\t\t\t.n_bits = %u,\n"
10692                         "\t\t\t\t.offset = %u,\n"
10693                         "\t\t\t},\n",
10694                         instr->meter.length.struct_id,
10695                         instr->meter.length.n_bits,
10696                         instr->meter.length.offset);
10697
10698                 /* instr.meter.color_in / instr.meter.color_in_val. */
10699                 if (!color_in_imm)
10700                         fprintf(f,
10701                                 "\t\t\t.color_in = {\n"
10702                                 "\t\t\t\t.struct_id = %u,\n"
10703                                 "\t\t\t\t.n_bits = %u,\n"
10704                                 "\t\t\t\t.offset = %u,\n"
10705                                 "\t\t\t},\n",
10706                                 instr->meter.color_in.struct_id,
10707                                 instr->meter.color_in.n_bits,
10708                                 instr->meter.color_in.offset);
10709                 else
10710                         fprintf(f,
10711                                 "\t\t\t.color_in_val = %u,\n",
10712                                 (uint32_t)instr->meter.color_in_val);
10713
10714                 /* instr.meter.color_out. */
10715                 fprintf(f,
10716                         "\t\t\t.color_out = {\n"
10717                         "\t\t\t\t.struct_id = %u,\n"
10718                         "\t\t\t\t.n_bits = %u,\n"
10719                         "\t\t\t\t.offset = %u,\n"
10720                         "\t\t\t},\n",
10721                         instr->meter.color_out.struct_id,
10722                         instr->meter.color_out.n_bits,
10723                         instr->meter.color_out.offset);
10724         }
10725
10726         /* instr.meter and instr - closing curly braces. */
10727         fprintf(f,
10728                 "\t\t},\n"
10729                 "\t},\n");
10730 }
10731
10732 static void
10733 instr_table_export(struct instruction *instr,
10734                 FILE *f)
10735 {
10736         fprintf(f,
10737                 "\t{\n"
10738                 "\t\t.type = %s,\n"
10739                 "\t\t.table = {\n"
10740                 "\t\t\t.table_id = %u,\n"
10741                 "\t\t},\n"
10742                 "\t},\n",
10743                 instr_type_to_name(instr),
10744                 instr->table.table_id);
10745 }
10746
10747 static void
10748 instr_learn_export(struct instruction *instr, FILE *f)
10749 {
10750         fprintf(f,
10751                 "\t{\n"
10752                 "\t\t.type = %s,\n"
10753                 "\t\t.learn = {\n"
10754                 "\t\t\t\t.action_id = %u,\n"
10755                 "\t\t},\n"
10756                 "\t},\n",
10757                 instr_type_to_name(instr),
10758                 instr->learn.action_id);
10759 }
10760
10761 static void
10762 instr_forget_export(struct instruction *instr, FILE *f)
10763 {
10764         fprintf(f,
10765                 "\t{\n"
10766                 "\t\t.type = %s,\n"
10767                 "\t},\n",
10768                 instr_type_to_name(instr));
10769 }
10770
10771 static void
10772 instr_extern_export(struct instruction *instr, FILE *f)
10773 {
10774         if (instr->type == INSTR_EXTERN_OBJ)
10775                 fprintf(f,
10776                         "\t{\n"
10777                         "\t\t.type = %s,\n"
10778                         "\t\t.ext_obj = {\n"
10779                         "\t\t\t.ext_obj_id = %u,\n"
10780                         "\t\t\t.func_id = %u,\n"
10781                         "\t\t},\n"
10782                         "\t},\n",
10783                         instr_type_to_name(instr),
10784                         instr->ext_obj.ext_obj_id,
10785                         instr->ext_obj.func_id);
10786         else
10787                 fprintf(f,
10788                         "\t{\n"
10789                         "\t\t.type = %s,\n"
10790                         "\t\t.ext_func = {\n"
10791                         "\t\t\t.ext_func_id = %u,\n"
10792                         "\t\t},\n"
10793                         "\t},\n",
10794                         instr_type_to_name(instr),
10795                         instr->ext_func.ext_func_id);
10796 }
10797
10798 static void
10799 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10800 {
10801         fprintf(f,
10802                 "\t{\n"
10803                 "\t\t.type = %s,\n"
10804                 "\t\t.jmp = {\n"
10805                 "\t\t\t.ip = NULL,\n",
10806                 instr_type_to_name(instr));
10807
10808         switch (instr->type) {
10809         case INSTR_JMP_VALID:
10810         case INSTR_JMP_INVALID:
10811                 fprintf(f,
10812                         "\t\t\t.header_id = %u,\n",
10813                         instr->jmp.header_id);
10814                 break;
10815
10816         case INSTR_JMP_ACTION_HIT:
10817         case INSTR_JMP_ACTION_MISS:
10818                 fprintf(f,
10819                         "\t\t\t.action_id = %u,\n",
10820                         instr->jmp.action_id);
10821                 break;
10822
10823         case INSTR_JMP_EQ:
10824         case INSTR_JMP_EQ_MH:
10825         case INSTR_JMP_EQ_HM:
10826         case INSTR_JMP_EQ_HH:
10827         case INSTR_JMP_NEQ:
10828         case INSTR_JMP_NEQ_MH:
10829         case INSTR_JMP_NEQ_HM:
10830         case INSTR_JMP_NEQ_HH:
10831         case INSTR_JMP_LT:
10832         case INSTR_JMP_LT_MH:
10833         case INSTR_JMP_LT_HM:
10834         case INSTR_JMP_LT_HH:
10835         case INSTR_JMP_GT:
10836         case INSTR_JMP_GT_MH:
10837         case INSTR_JMP_GT_HM:
10838         case INSTR_JMP_GT_HH:
10839                 fprintf(f,
10840                         "\t\t\t.a = {\n"
10841                         "\t\t\t\t.struct_id = %u,\n"
10842                         "\t\t\t\t.n_bits = %u,\n"
10843                         "\t\t\t\t.offset = %u,\n"
10844                         "\t\t\t},\n"
10845                         "\t\t\t.b = {\n"
10846                         "\t\t\t\t.struct_id = %u,\n"
10847                         "\t\t\t\t.n_bits = %u,\n"
10848                         "\t\t\t\t.offset = %u,\n"
10849                         "\t\t\t},\n",
10850                         instr->jmp.a.struct_id,
10851                         instr->jmp.a.n_bits,
10852                         instr->jmp.a.offset,
10853                         instr->jmp.b.struct_id,
10854                         instr->jmp.b.n_bits,
10855                         instr->jmp.b.offset);
10856                 break;
10857
10858         case INSTR_JMP_EQ_I:
10859         case INSTR_JMP_NEQ_I:
10860         case INSTR_JMP_LT_MI:
10861         case INSTR_JMP_LT_HI:
10862         case INSTR_JMP_GT_MI:
10863         case INSTR_JMP_GT_HI:
10864                 fprintf(f,
10865                         "\t\t\t.a = {\n"
10866                         "\t\t\t\t.struct_id = %u,\n"
10867                         "\t\t\t\t.n_bits = %u,\n"
10868                         "\t\t\t\t.offset = %u,\n"
10869                         "\t\t\t}\n,"
10870                         "\t\t\t.b_val = %" PRIu64 ",\n",
10871                         instr->jmp.a.struct_id,
10872                         instr->jmp.a.n_bits,
10873                         instr->jmp.a.offset,
10874                         instr->jmp.b_val);
10875                 break;
10876
10877         default:
10878                 break;
10879         }
10880
10881         fprintf(f,
10882                 "\t\t},\n"
10883                 "\t},\n");
10884 }
10885
10886 static void
10887 instr_return_export(struct instruction *instr,
10888                 FILE *f)
10889 {
10890         fprintf(f,
10891                 "\t{\n"
10892                 "\t\t.type = %s,\n",
10893                 instr_type_to_name(instr));
10894
10895         fprintf(f,
10896                 "\t},\n");
10897 }
10898
10899 static instruction_export_t export_table[] = {
10900         [INSTR_RX] = instr_io_export,
10901
10902         [INSTR_TX] = instr_io_export,
10903         [INSTR_TX_I] = instr_io_export,
10904         [INSTR_DROP] = instr_io_export,
10905
10906         [INSTR_HDR_EXTRACT] = instr_io_export,
10907         [INSTR_HDR_EXTRACT2] = instr_io_export,
10908         [INSTR_HDR_EXTRACT3] = instr_io_export,
10909         [INSTR_HDR_EXTRACT4] = instr_io_export,
10910         [INSTR_HDR_EXTRACT5] = instr_io_export,
10911         [INSTR_HDR_EXTRACT6] = instr_io_export,
10912         [INSTR_HDR_EXTRACT7] = instr_io_export,
10913         [INSTR_HDR_EXTRACT8] = instr_io_export,
10914
10915         [INSTR_HDR_EXTRACT_M] = instr_io_export,
10916
10917         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10918
10919         [INSTR_HDR_EMIT] = instr_io_export,
10920         [INSTR_HDR_EMIT_TX] = instr_io_export,
10921         [INSTR_HDR_EMIT2_TX] = instr_io_export,
10922         [INSTR_HDR_EMIT3_TX] = instr_io_export,
10923         [INSTR_HDR_EMIT4_TX] = instr_io_export,
10924         [INSTR_HDR_EMIT5_TX] = instr_io_export,
10925         [INSTR_HDR_EMIT6_TX] = instr_io_export,
10926         [INSTR_HDR_EMIT7_TX] = instr_io_export,
10927         [INSTR_HDR_EMIT8_TX] = instr_io_export,
10928
10929         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10930         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10931
10932         [INSTR_MOV] = instr_mov_export,
10933         [INSTR_MOV_MH] = instr_mov_export,
10934         [INSTR_MOV_HM] = instr_mov_export,
10935         [INSTR_MOV_HH] = instr_mov_export,
10936         [INSTR_MOV_I] = instr_mov_export,
10937
10938         [INSTR_DMA_HT]  = instr_dma_ht_export,
10939         [INSTR_DMA_HT2] = instr_dma_ht_export,
10940         [INSTR_DMA_HT3] = instr_dma_ht_export,
10941         [INSTR_DMA_HT4] = instr_dma_ht_export,
10942         [INSTR_DMA_HT5] = instr_dma_ht_export,
10943         [INSTR_DMA_HT6] = instr_dma_ht_export,
10944         [INSTR_DMA_HT7] = instr_dma_ht_export,
10945         [INSTR_DMA_HT8] = instr_dma_ht_export,
10946
10947         [INSTR_ALU_ADD] = instr_alu_export,
10948         [INSTR_ALU_ADD_MH] = instr_alu_export,
10949         [INSTR_ALU_ADD_HM] = instr_alu_export,
10950         [INSTR_ALU_ADD_HH] = instr_alu_export,
10951         [INSTR_ALU_ADD_MI] = instr_alu_export,
10952         [INSTR_ALU_ADD_HI] = instr_alu_export,
10953
10954         [INSTR_ALU_SUB] = instr_alu_export,
10955         [INSTR_ALU_SUB_MH] = instr_alu_export,
10956         [INSTR_ALU_SUB_HM] = instr_alu_export,
10957         [INSTR_ALU_SUB_HH] = instr_alu_export,
10958         [INSTR_ALU_SUB_MI] = instr_alu_export,
10959         [INSTR_ALU_SUB_HI] = instr_alu_export,
10960
10961         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10962         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10963         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10964         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10965
10966         [INSTR_ALU_AND] = instr_alu_export,
10967         [INSTR_ALU_AND_MH] = instr_alu_export,
10968         [INSTR_ALU_AND_HM] = instr_alu_export,
10969         [INSTR_ALU_AND_HH] = instr_alu_export,
10970         [INSTR_ALU_AND_I] = instr_alu_export,
10971
10972         [INSTR_ALU_OR] = instr_alu_export,
10973         [INSTR_ALU_OR_MH] = instr_alu_export,
10974         [INSTR_ALU_OR_HM] = instr_alu_export,
10975         [INSTR_ALU_OR_HH] = instr_alu_export,
10976         [INSTR_ALU_OR_I] = instr_alu_export,
10977
10978         [INSTR_ALU_XOR] = instr_alu_export,
10979         [INSTR_ALU_XOR_MH] = instr_alu_export,
10980         [INSTR_ALU_XOR_HM] = instr_alu_export,
10981         [INSTR_ALU_XOR_HH] = instr_alu_export,
10982         [INSTR_ALU_XOR_I] = instr_alu_export,
10983
10984         [INSTR_ALU_SHL] = instr_alu_export,
10985         [INSTR_ALU_SHL_MH] = instr_alu_export,
10986         [INSTR_ALU_SHL_HM] = instr_alu_export,
10987         [INSTR_ALU_SHL_HH] = instr_alu_export,
10988         [INSTR_ALU_SHL_MI] = instr_alu_export,
10989         [INSTR_ALU_SHL_HI] = instr_alu_export,
10990
10991         [INSTR_ALU_SHR] = instr_alu_export,
10992         [INSTR_ALU_SHR_MH] = instr_alu_export,
10993         [INSTR_ALU_SHR_HM] = instr_alu_export,
10994         [INSTR_ALU_SHR_HH] = instr_alu_export,
10995         [INSTR_ALU_SHR_MI] = instr_alu_export,
10996         [INSTR_ALU_SHR_HI] = instr_alu_export,
10997
10998         [INSTR_REGPREFETCH_RH] = instr_reg_export,
10999         [INSTR_REGPREFETCH_RM] = instr_reg_export,
11000         [INSTR_REGPREFETCH_RI] = instr_reg_export,
11001
11002         [INSTR_REGRD_HRH] = instr_reg_export,
11003         [INSTR_REGRD_HRM] = instr_reg_export,
11004         [INSTR_REGRD_MRH] = instr_reg_export,
11005         [INSTR_REGRD_MRM] = instr_reg_export,
11006         [INSTR_REGRD_HRI] = instr_reg_export,
11007         [INSTR_REGRD_MRI] = instr_reg_export,
11008
11009         [INSTR_REGWR_RHH] = instr_reg_export,
11010         [INSTR_REGWR_RHM] = instr_reg_export,
11011         [INSTR_REGWR_RMH] = instr_reg_export,
11012         [INSTR_REGWR_RMM] = instr_reg_export,
11013         [INSTR_REGWR_RHI] = instr_reg_export,
11014         [INSTR_REGWR_RMI] = instr_reg_export,
11015         [INSTR_REGWR_RIH] = instr_reg_export,
11016         [INSTR_REGWR_RIM] = instr_reg_export,
11017         [INSTR_REGWR_RII] = instr_reg_export,
11018
11019         [INSTR_REGADD_RHH] = instr_reg_export,
11020         [INSTR_REGADD_RHM] = instr_reg_export,
11021         [INSTR_REGADD_RMH] = instr_reg_export,
11022         [INSTR_REGADD_RMM] = instr_reg_export,
11023         [INSTR_REGADD_RHI] = instr_reg_export,
11024         [INSTR_REGADD_RMI] = instr_reg_export,
11025         [INSTR_REGADD_RIH] = instr_reg_export,
11026         [INSTR_REGADD_RIM] = instr_reg_export,
11027         [INSTR_REGADD_RII] = instr_reg_export,
11028
11029         [INSTR_METPREFETCH_H] = instr_meter_export,
11030         [INSTR_METPREFETCH_M] = instr_meter_export,
11031         [INSTR_METPREFETCH_I] = instr_meter_export,
11032
11033         [INSTR_METER_HHM] = instr_meter_export,
11034         [INSTR_METER_HHI] = instr_meter_export,
11035         [INSTR_METER_HMM] = instr_meter_export,
11036         [INSTR_METER_HMI] = instr_meter_export,
11037         [INSTR_METER_MHM] = instr_meter_export,
11038         [INSTR_METER_MHI] = instr_meter_export,
11039         [INSTR_METER_MMM] = instr_meter_export,
11040         [INSTR_METER_MMI] = instr_meter_export,
11041         [INSTR_METER_IHM] = instr_meter_export,
11042         [INSTR_METER_IHI] = instr_meter_export,
11043         [INSTR_METER_IMM] = instr_meter_export,
11044         [INSTR_METER_IMI] = instr_meter_export,
11045
11046         [INSTR_TABLE] = instr_table_export,
11047         [INSTR_TABLE_AF] = instr_table_export,
11048         [INSTR_SELECTOR] = instr_table_export,
11049         [INSTR_LEARNER] = instr_table_export,
11050         [INSTR_LEARNER_AF] = instr_table_export,
11051
11052         [INSTR_LEARNER_LEARN] = instr_learn_export,
11053         [INSTR_LEARNER_FORGET] = instr_forget_export,
11054
11055         [INSTR_EXTERN_OBJ] = instr_extern_export,
11056         [INSTR_EXTERN_FUNC] = instr_extern_export,
11057
11058         [INSTR_JMP] = instr_jmp_export,
11059         [INSTR_JMP_VALID] = instr_jmp_export,
11060         [INSTR_JMP_INVALID] = instr_jmp_export,
11061         [INSTR_JMP_HIT] = instr_jmp_export,
11062         [INSTR_JMP_MISS] = instr_jmp_export,
11063         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11064         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11065
11066         [INSTR_JMP_EQ] = instr_jmp_export,
11067         [INSTR_JMP_EQ_MH] = instr_jmp_export,
11068         [INSTR_JMP_EQ_HM] = instr_jmp_export,
11069         [INSTR_JMP_EQ_HH] = instr_jmp_export,
11070         [INSTR_JMP_EQ_I] = instr_jmp_export,
11071
11072         [INSTR_JMP_NEQ] = instr_jmp_export,
11073         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11074         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11075         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11076         [INSTR_JMP_NEQ_I] = instr_jmp_export,
11077
11078         [INSTR_JMP_LT] = instr_jmp_export,
11079         [INSTR_JMP_LT_MH] = instr_jmp_export,
11080         [INSTR_JMP_LT_HM] = instr_jmp_export,
11081         [INSTR_JMP_LT_HH] = instr_jmp_export,
11082         [INSTR_JMP_LT_MI] = instr_jmp_export,
11083         [INSTR_JMP_LT_HI] = instr_jmp_export,
11084
11085         [INSTR_JMP_GT] = instr_jmp_export,
11086         [INSTR_JMP_GT_MH] = instr_jmp_export,
11087         [INSTR_JMP_GT_HM] = instr_jmp_export,
11088         [INSTR_JMP_GT_HH] = instr_jmp_export,
11089         [INSTR_JMP_GT_MI] = instr_jmp_export,
11090         [INSTR_JMP_GT_HI] = instr_jmp_export,
11091
11092         [INSTR_RETURN] = instr_return_export,
11093 };
11094
11095 static void
11096 action_data_codegen(struct action *a, FILE *f)
11097 {
11098         uint32_t i;
11099
11100         fprintf(f,
11101                 "static const struct instruction action_%s_instructions[] = {\n",
11102                 a->name);
11103
11104         for (i = 0; i < a->n_instructions; i++) {
11105                 struct instruction *instr = &a->instructions[i];
11106                 instruction_export_t func = export_table[instr->type];
11107
11108                 func(instr, f);
11109         }
11110
11111         fprintf(f, "};\n");
11112 }
11113
11114 static const char *
11115 instr_type_to_func(struct instruction *instr)
11116 {
11117         switch (instr->type) {
11118         case INSTR_RX: return NULL;
11119
11120         case INSTR_TX: return "__instr_tx_exec";
11121         case INSTR_TX_I: return "__instr_tx_i_exec";
11122         case INSTR_DROP: return "__instr_drop_exec";
11123
11124         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11125         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11126         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11127         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11128         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11129         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11130         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11131         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11132
11133         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11134
11135         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11136
11137         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11138         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11139         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11140         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11141         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11142         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11143         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11144         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11145         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11146
11147         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11148         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11149
11150         case INSTR_MOV: return "__instr_mov_exec";
11151         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11152         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11153         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11154         case INSTR_MOV_I: return "__instr_mov_i_exec";
11155
11156         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11157         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11158         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11159         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11160         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11161         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11162         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11163         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11164
11165         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11166         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11167         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11168         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11169         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11170         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11171
11172         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11173         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11174         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11175         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11176         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11177         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11178
11179         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11180         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11181         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11182         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11183
11184         case INSTR_ALU_AND: return "__instr_alu_and_exec";
11185         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11186         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11187         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11188         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11189
11190         case INSTR_ALU_OR: return "__instr_alu_or_exec";
11191         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11192         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11193         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11194         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11195
11196         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11197         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11198         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11199         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11200         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11201
11202         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11203         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11204         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11205         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11206         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11207         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11208
11209         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11210         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11211         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11212         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11213         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11214         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11215
11216         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11217         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11218         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11219
11220         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11221         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11222         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11223         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11224         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11225         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11226
11227         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11228         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11229         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11230         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11231         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11232         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11233         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11234         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11235         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11236
11237         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11238         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11239         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11240         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11241         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11242         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11243         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11244         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11245         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11246
11247         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11248         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11249         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11250
11251         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11252         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11253         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11254         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11255         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11256         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11257         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11258         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11259         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11260         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11261         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11262         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11263
11264         case INSTR_TABLE: return NULL;
11265         case INSTR_TABLE_AF: return NULL;
11266         case INSTR_SELECTOR: return NULL;
11267         case INSTR_LEARNER: return NULL;
11268         case INSTR_LEARNER_AF: return NULL;
11269
11270         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11271         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11272
11273         case INSTR_EXTERN_OBJ: return NULL;
11274         case INSTR_EXTERN_FUNC: return NULL;
11275
11276         case INSTR_JMP: return NULL;
11277         case INSTR_JMP_VALID: return NULL;
11278         case INSTR_JMP_INVALID: return NULL;
11279         case INSTR_JMP_HIT: return NULL;
11280         case INSTR_JMP_MISS: return NULL;
11281         case INSTR_JMP_ACTION_HIT: return NULL;
11282         case INSTR_JMP_ACTION_MISS: return NULL;
11283         case INSTR_JMP_EQ: return NULL;
11284         case INSTR_JMP_EQ_MH: return NULL;
11285         case INSTR_JMP_EQ_HM: return NULL;
11286         case INSTR_JMP_EQ_HH: return NULL;
11287         case INSTR_JMP_EQ_I: return NULL;
11288         case INSTR_JMP_NEQ: return NULL;
11289         case INSTR_JMP_NEQ_MH: return NULL;
11290         case INSTR_JMP_NEQ_HM: return NULL;
11291         case INSTR_JMP_NEQ_HH: return NULL;
11292         case INSTR_JMP_NEQ_I: return NULL;
11293         case INSTR_JMP_LT: return NULL;
11294         case INSTR_JMP_LT_MH: return NULL;
11295         case INSTR_JMP_LT_HM: return NULL;
11296         case INSTR_JMP_LT_HH: return NULL;
11297         case INSTR_JMP_LT_MI: return NULL;
11298         case INSTR_JMP_LT_HI: return NULL;
11299         case INSTR_JMP_GT: return NULL;
11300         case INSTR_JMP_GT_MH: return NULL;
11301         case INSTR_JMP_GT_HM: return NULL;
11302         case INSTR_JMP_GT_HH: return NULL;
11303         case INSTR_JMP_GT_MI: return NULL;
11304         case INSTR_JMP_GT_HI: return NULL;
11305
11306         case INSTR_RETURN: return NULL;
11307
11308         default: return NULL;
11309         }
11310 }
11311
11312 static void
11313 action_instr_does_tx_codegen(struct action *a,
11314                         uint32_t instr_pos,
11315                         struct instruction *instr,
11316                         FILE *f)
11317 {
11318         fprintf(f,
11319                 "%s(p, t, &action_%s_instructions[%u]);\n"
11320                 "\tthread_ip_reset(p, t);\n"
11321                 "\tinstr_rx_exec(p);\n"
11322                 "\treturn;\n",
11323                 instr_type_to_func(instr),
11324                 a->name,
11325                 instr_pos);
11326 }
11327
11328 static void
11329 action_instr_extern_obj_codegen(struct action *a,
11330                                 uint32_t instr_pos,
11331                                 FILE *f)
11332 {
11333         fprintf(f,
11334                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11335                 a->name,
11336                 instr_pos);
11337 }
11338
11339 static void
11340 action_instr_extern_func_codegen(struct action *a,
11341                                  uint32_t instr_pos,
11342                                  FILE *f)
11343 {
11344         fprintf(f,
11345                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11346                 a->name,
11347                 instr_pos);
11348 }
11349
11350 static void
11351 action_instr_jmp_codegen(struct action *a,
11352                          uint32_t instr_pos,
11353                          struct instruction *instr,
11354                          struct instruction_data *data,
11355                          FILE *f)
11356 {
11357         switch (instr->type) {
11358         case INSTR_JMP:
11359                 fprintf(f,
11360                         "goto %s;\n",
11361                         data->jmp_label);
11362                 return;
11363
11364         case INSTR_JMP_VALID:
11365                 fprintf(f,
11366                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11367                         "\t\tgoto %s;\n",
11368                         a->name,
11369                         instr_pos,
11370                         data->jmp_label);
11371                 return;
11372
11373         case INSTR_JMP_INVALID:
11374                 fprintf(f,
11375                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11376                         "\t\tgoto %s;\n",
11377                         a->name,
11378                         instr_pos,
11379                         data->jmp_label);
11380                 return;
11381
11382         case INSTR_JMP_HIT:
11383                 fprintf(f,
11384                         "if (t->hit)\n"
11385                         "\t\tgoto %s;\n",
11386                         data->jmp_label);
11387                 return;
11388
11389         case INSTR_JMP_MISS:
11390                 fprintf(f,
11391                         "if (!t->hit)\n"
11392                         "\t\tgoto %s;\n",
11393                         data->jmp_label);
11394                 return;
11395
11396         case INSTR_JMP_ACTION_HIT:
11397                 fprintf(f,
11398                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11399                         "\t\tgoto %s;\n",
11400                         a->name,
11401                         instr_pos,
11402                         data->jmp_label);
11403                 return;
11404
11405         case INSTR_JMP_ACTION_MISS:
11406                 fprintf(f,
11407                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11408                         "\t\tgoto %s;\n",
11409                         a->name,
11410                         instr_pos,
11411                         data->jmp_label);
11412                 return;
11413
11414         case INSTR_JMP_EQ:
11415                 fprintf(f,
11416                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11417                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11418                         "\t\tgoto %s;\n",
11419                         a->name,
11420                         instr_pos,
11421                         a->name,
11422                         instr_pos,
11423                         data->jmp_label);
11424                 return;
11425
11426         case INSTR_JMP_EQ_MH:
11427                 fprintf(f,
11428                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11429                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11430                         "\t\tgoto %s;\n",
11431                         a->name,
11432                         instr_pos,
11433                         a->name,
11434                         instr_pos,
11435                         data->jmp_label);
11436                 return;
11437
11438         case INSTR_JMP_EQ_HM:
11439                 fprintf(f,
11440                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11441                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11442                         "\t\tgoto %s;\n",
11443                         a->name,
11444                         instr_pos,
11445                         a->name,
11446                         instr_pos,
11447                         data->jmp_label);
11448                 return;
11449
11450         case INSTR_JMP_EQ_HH:
11451                 fprintf(f,
11452                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11453                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11454                         "\t\tgoto %s;\n",
11455                         a->name,
11456                         instr_pos,
11457                         a->name,
11458                         instr_pos,
11459                         data->jmp_label);
11460                 return;
11461
11462         case INSTR_JMP_EQ_I:
11463                 fprintf(f,
11464                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11465                         "action_%s_instructions[%u].jmp.b_val)\n"
11466                         "\t\tgoto %s;\n",
11467                         a->name,
11468                         instr_pos,
11469                         a->name,
11470                         instr_pos,
11471                         data->jmp_label);
11472                 return;
11473
11474         case INSTR_JMP_NEQ:
11475                 fprintf(f,
11476                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11477                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11478                         "\t\tgoto %s;\n",
11479                         a->name,
11480                         instr_pos,
11481                         a->name,
11482                         instr_pos,
11483                         data->jmp_label);
11484                 return;
11485
11486         case INSTR_JMP_NEQ_MH:
11487                 fprintf(f,
11488                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11489                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11490                         "\t\tgoto %s;\n",
11491                         a->name,
11492                         instr_pos,
11493                         a->name,
11494                         instr_pos,
11495                         data->jmp_label);
11496                 return;
11497
11498         case INSTR_JMP_NEQ_HM:
11499                 fprintf(f,
11500                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11501                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11502                         "\t\tgoto %s;\n",
11503                         a->name,
11504                         instr_pos,
11505                         a->name,
11506                         instr_pos,
11507                         data->jmp_label);
11508                 return;
11509
11510         case INSTR_JMP_NEQ_HH:
11511                 fprintf(f,
11512                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11513                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11514                         "\t\tgoto %s;\n",
11515                         a->name,
11516                         instr_pos,
11517                         a->name,
11518                         instr_pos,
11519                         data->jmp_label);
11520                 return;
11521
11522         case INSTR_JMP_NEQ_I:
11523                 fprintf(f,
11524                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11525                         "action_%s_instructions[%u].jmp.b_val)\n"
11526                         "\t\tgoto %s;\n",
11527                         a->name,
11528                         instr_pos,
11529                         a->name,
11530                         instr_pos,
11531                         data->jmp_label);
11532                 return;
11533
11534         case INSTR_JMP_LT:
11535                 fprintf(f,
11536                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11537                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11538                         "\t\tgoto %s;\n",
11539                         a->name,
11540                         instr_pos,
11541                         a->name,
11542                         instr_pos,
11543                         data->jmp_label);
11544                 return;
11545
11546         case INSTR_JMP_LT_MH:
11547                 fprintf(f,
11548                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11549                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11550                         "\t\tgoto %s;\n",
11551                         a->name,
11552                         instr_pos,
11553                         a->name,
11554                         instr_pos,
11555                         data->jmp_label);
11556                 return;
11557
11558         case INSTR_JMP_LT_HM:
11559                 fprintf(f,
11560                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11561                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11562                         "\t\tgoto %s;\n",
11563                         a->name,
11564                         instr_pos,
11565                         a->name,
11566                         instr_pos,
11567                         data->jmp_label);
11568                 return;
11569
11570         case INSTR_JMP_LT_HH:
11571                 fprintf(f,
11572                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11573                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11574                         "\t\tgoto %s;\n",
11575                         a->name,
11576                         instr_pos,
11577                         a->name,
11578                         instr_pos,
11579                         data->jmp_label);
11580                 return;
11581
11582         case INSTR_JMP_LT_MI:
11583                 fprintf(f,
11584                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11585                         "action_%s_instructions[%u].jmp.b_val)\n"
11586                         "\t\tgoto %s;\n",
11587                         a->name,
11588                         instr_pos,
11589                         a->name,
11590                         instr_pos,
11591                         data->jmp_label);
11592                 return;
11593
11594         case INSTR_JMP_LT_HI:
11595                 fprintf(f,
11596                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11597                         "action_%s_instructions[%u].jmp.b_val)\n"
11598                         "\t\tgoto %s;\n",
11599                         a->name,
11600                         instr_pos,
11601                         a->name,
11602                         instr_pos,
11603                         data->jmp_label);
11604                 return;
11605
11606         case INSTR_JMP_GT:
11607                 fprintf(f,
11608                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11609                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11610                         "\t\tgoto %s;\n",
11611                         a->name,
11612                         instr_pos,
11613                         a->name,
11614                         instr_pos,
11615                         data->jmp_label);
11616                 return;
11617
11618         case INSTR_JMP_GT_MH:
11619                 fprintf(f,
11620                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11621                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11622                         "\t\tgoto %s;\n",
11623                         a->name,
11624                         instr_pos,
11625                         a->name,
11626                         instr_pos,
11627                         data->jmp_label);
11628                 return;
11629
11630         case INSTR_JMP_GT_HM:
11631                 fprintf(f,
11632                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11633                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11634                         "\t\tgoto %s;\n",
11635                         a->name,
11636                         instr_pos,
11637                         a->name,
11638                         instr_pos,
11639                         data->jmp_label);
11640                 return;
11641
11642         case INSTR_JMP_GT_HH:
11643                 fprintf(f,
11644                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11645                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11646                         "\t\tgoto %s;\n",
11647                         a->name,
11648                         instr_pos,
11649                         a->name,
11650                         instr_pos,
11651                         data->jmp_label);
11652                 return;
11653
11654         case INSTR_JMP_GT_MI:
11655                 fprintf(f,
11656                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11657                         "action_%s_instructions[%u].jmp.b_val)\n"
11658                         "\t\tgoto %s;\n",
11659                         a->name,
11660                         instr_pos,
11661                         a->name,
11662                         instr_pos,
11663                         data->jmp_label);
11664                 return;
11665
11666         case INSTR_JMP_GT_HI:
11667                 fprintf(f,
11668                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11669                         "action_%s_instructions[%u].jmp.b_val)\n"
11670                         "\t\tgoto %s;\n",
11671                         a->name,
11672                         instr_pos,
11673                         a->name,
11674                         instr_pos,
11675                         data->jmp_label);
11676                 return;
11677
11678         default:
11679                 return;
11680         }
11681 }
11682
11683 static void
11684 action_instr_return_codegen(FILE *f)
11685 {
11686         fprintf(f,
11687                 "return;\n");
11688 }
11689
11690 static void
11691 action_instr_codegen(struct action *a, FILE *f)
11692 {
11693         uint32_t i;
11694
11695         fprintf(f,
11696                 "void\n"
11697                 "action_%s_run(struct rte_swx_pipeline *p)\n"
11698                 "{\n"
11699                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11700                 "\n",
11701                 a->name);
11702
11703         for (i = 0; i < a->n_instructions; i++) {
11704                 struct instruction *instr = &a->instructions[i];
11705                 struct instruction_data *data = &a->instruction_data[i];
11706
11707                 /* Label, if present. */
11708                 if (data->label[0])
11709                         fprintf(f, "\n%s : ", data->label);
11710                 else
11711                         fprintf(f, "\n\t");
11712
11713                 /* TX instruction type. */
11714                 if (instruction_does_tx(instr)) {
11715                         action_instr_does_tx_codegen(a, i, instr, f);
11716                         continue;
11717                 }
11718
11719                 /* Extern object/function instruction type. */
11720                 if (instr->type == INSTR_EXTERN_OBJ) {
11721                         action_instr_extern_obj_codegen(a, i, f);
11722                         continue;
11723                 }
11724
11725                 if (instr->type == INSTR_EXTERN_FUNC) {
11726                         action_instr_extern_func_codegen(a, i, f);
11727                         continue;
11728                 }
11729
11730                 /* Jump instruction type. */
11731                 if (instruction_is_jmp(instr)) {
11732                         action_instr_jmp_codegen(a, i, instr, data, f);
11733                         continue;
11734                 }
11735
11736                 /* Return instruction type. */
11737                 if (instr->type == INSTR_RETURN) {
11738                         action_instr_return_codegen(f);
11739                         continue;
11740                 }
11741
11742                 /* Any other instruction type. */
11743                 fprintf(f,
11744                         "%s(p, t, &action_%s_instructions[%u]);\n",
11745                         instr_type_to_func(instr),
11746                         a->name,
11747                         i);
11748         }
11749
11750         fprintf(f, "}\n\n");
11751 }
11752
11753 struct instruction_group {
11754         TAILQ_ENTRY(instruction_group) node;
11755
11756         uint32_t group_id;
11757
11758         uint32_t first_instr_id;
11759
11760         uint32_t last_instr_id;
11761
11762         instr_exec_t func;
11763 };
11764
11765 TAILQ_HEAD(instruction_group_list, instruction_group);
11766
11767 static struct instruction_group *
11768 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11769 {
11770         struct instruction_group *g;
11771
11772         TAILQ_FOREACH(g, igl, node)
11773                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11774                         return g;
11775
11776         return NULL;
11777 }
11778
11779 static void
11780 instruction_group_list_free(struct instruction_group_list *igl)
11781 {
11782         if (!igl)
11783                 return;
11784
11785         for ( ; ; ) {
11786                 struct instruction_group *g;
11787
11788                 g = TAILQ_FIRST(igl);
11789                 if (!g)
11790                         break;
11791
11792                 TAILQ_REMOVE(igl, g, node);
11793                 free(g);
11794         }
11795
11796         free(igl);
11797 }
11798
11799 static struct instruction_group_list *
11800 instruction_group_list_create(struct rte_swx_pipeline *p)
11801 {
11802         struct instruction_group_list *igl = NULL;
11803         struct instruction_group *g = NULL;
11804         uint32_t n_groups = 0, i;
11805
11806         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11807                 goto error;
11808
11809         /* List init. */
11810         igl = calloc(1, sizeof(struct instruction_group_list));
11811         if (!igl)
11812                 goto error;
11813
11814         TAILQ_INIT(igl);
11815
11816         /* Allocate the first group. */
11817         g = calloc(1, sizeof(struct instruction_group));
11818         if (!g)
11819                 goto error;
11820
11821         /* Iteration 1: Separate the instructions into groups based on the thread yield
11822          * instructions. Do not worry about the jump instructions at this point.
11823          */
11824         for (i = 0; i < p->n_instructions; i++) {
11825                 struct instruction *instr = &p->instructions[i];
11826
11827                 /* Check for thread yield instructions. */
11828                 if (!instruction_does_thread_yield(instr))
11829                         continue;
11830
11831                 /* If the current group contains at least one instruction, then finalize it (with
11832                  * the previous instruction), add it to the list and allocate a new group (that
11833                  * starts with the current instruction).
11834                  */
11835                 if (i - g->first_instr_id) {
11836                         /* Finalize the group. */
11837                         g->last_instr_id = i - 1;
11838
11839                         /* Add the group to the list. Advance the number of groups. */
11840                         TAILQ_INSERT_TAIL(igl, g, node);
11841                         n_groups++;
11842
11843                         /* Allocate a new group. */
11844                         g = calloc(1, sizeof(struct instruction_group));
11845                         if (!g)
11846                                 goto error;
11847
11848                         /* Initialize the new group. */
11849                         g->group_id = n_groups;
11850                         g->first_instr_id = i;
11851                 }
11852
11853                 /* Finalize the current group (with the current instruction, therefore this group
11854                  * contains just the current thread yield instruction), add it to the list and
11855                  * allocate a new group (that starts with the next instruction).
11856                  */
11857
11858                 /* Finalize the group. */
11859                 g->last_instr_id = i;
11860
11861                 /* Add the group to the list. Advance the number of groups. */
11862                 TAILQ_INSERT_TAIL(igl, g, node);
11863                 n_groups++;
11864
11865                 /* Allocate a new group. */
11866                 g = calloc(1, sizeof(struct instruction_group));
11867                 if (!g)
11868                         goto error;
11869
11870                 /* Initialize the new group. */
11871                 g->group_id = n_groups;
11872                 g->first_instr_id = i + 1;
11873         }
11874
11875         /* Handle the last group. */
11876         if (i - g->first_instr_id) {
11877                 /* Finalize the group. */
11878                 g->last_instr_id = i - 1;
11879
11880                 /* Add the group to the list. Advance the number of groups. */
11881                 TAILQ_INSERT_TAIL(igl, g, node);
11882                 n_groups++;
11883         } else
11884                 free(g);
11885
11886         g = NULL;
11887
11888         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11889          * the destination of a jump instruction located in a different group ("far jump"), then the
11890          * current group has to be split, so that the instruction representing the far jump
11891          * destination is at the start of its group.
11892          */
11893         for ( ; ; ) {
11894                 int is_modified = 0;
11895
11896                 for (i = 0; i < p->n_instructions; i++) {
11897                         struct instruction_data *data = &p->instruction_data[i];
11898                         struct instruction_group *g;
11899                         uint32_t j;
11900
11901                         /* Continue when the current instruction is not a jump destination. */
11902                         if (!data->n_users)
11903                                 continue;
11904
11905                         g = instruction_group_list_group_find(igl, i);
11906                         if (!g)
11907                                 goto error;
11908
11909                         /* Find out all the jump instructions with this destination. */
11910                         for (j = 0; j < p->n_instructions; j++) {
11911                                 struct instruction *jmp_instr = &p->instructions[j];
11912                                 struct instruction_data *jmp_data = &p->instruction_data[j];
11913                                 struct instruction_group *jmp_g, *new_g;
11914
11915                                 /* Continue when not a jump instruction. Even when jump instruction,
11916                                  * continue when the jump destination is not this instruction.
11917                                  */
11918                                 if (!instruction_is_jmp(jmp_instr) ||
11919                                     strcmp(jmp_data->jmp_label, data->label))
11920                                         continue;
11921
11922                                 jmp_g = instruction_group_list_group_find(igl, j);
11923                                 if (!jmp_g)
11924                                         goto error;
11925
11926                                 /* Continue when both the jump instruction and the jump destination
11927                                  * instruction are in the same group. Even when in different groups,
11928                                  * still continue if the jump destination instruction is already the
11929                                  * first instruction of its group.
11930                                  */
11931                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11932                                         continue;
11933
11934                                 /* Split the group of the current jump destination instruction to
11935                                  * make this instruction the first instruction of a new group.
11936                                  */
11937                                 new_g = calloc(1, sizeof(struct instruction_group));
11938                                 if (!new_g)
11939                                         goto error;
11940
11941                                 new_g->group_id = n_groups;
11942                                 new_g->first_instr_id = i;
11943                                 new_g->last_instr_id = g->last_instr_id;
11944
11945                                 g->last_instr_id = i - 1;
11946
11947                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11948                                 n_groups++;
11949                                 is_modified = 1;
11950
11951                                 /* The decision to split this group (to make the current instruction
11952                                  * the first instruction of a new group) is already taken and fully
11953                                  * implemented, so no need to search for more reasons to do it.
11954                                  */
11955                                 break;
11956                         }
11957                 }
11958
11959                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11960                  * previously considered local (i.e. the jump destination is in the same group as
11961                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11962                  * different group than the jump instruction). Wost case scenario: each instruction
11963                  * that is a jump destination ends up as the first instruction of its group.
11964                  */
11965                 if (!is_modified)
11966                         break;
11967         }
11968
11969         /* Re-assign the group IDs to be in incremental order. */
11970         i = 0;
11971         TAILQ_FOREACH(g, igl, node) {
11972                 g->group_id = i;
11973
11974                 i++;
11975         }
11976
11977         return igl;
11978
11979 error:
11980         instruction_group_list_free(igl);
11981
11982         free(g);
11983
11984         return NULL;
11985 }
11986
11987 static void
11988 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11989                                uint32_t instr_pos,
11990                                struct instruction *instr,
11991                                FILE *f)
11992 {
11993         fprintf(f,
11994                 "%s(p, t, &pipeline_instructions[%u]);\n"
11995                 "\tthread_ip_reset(p, t);\n"
11996                 "\tinstr_rx_exec(p);\n"
11997                 "\treturn;\n",
11998                 instr_type_to_func(instr),
11999                 instr_pos);
12000 }
12001
12002 static int
12003 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12004                            struct instruction_group_list *igl,
12005                            uint32_t jmp_instr_id,
12006                            struct instruction *jmp_instr,
12007                            struct instruction_data *jmp_data,
12008                            FILE *f)
12009 {
12010         struct instruction_group *jmp_g, *g;
12011         struct instruction_data *data;
12012         uint32_t instr_id;
12013
12014         switch (jmp_instr->type) {
12015         case INSTR_JMP:
12016                 break;
12017
12018         case INSTR_JMP_VALID:
12019                 fprintf(f,
12020                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12021                         jmp_instr_id);
12022                 break;
12023
12024         case INSTR_JMP_INVALID:
12025                 fprintf(f,
12026                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12027                         jmp_instr_id);
12028                 break;
12029
12030         case INSTR_JMP_HIT:
12031                 fprintf(f,
12032                         "if (t->hit)\n");
12033                 break;
12034
12035         case INSTR_JMP_MISS:
12036                 fprintf(f,
12037                         "if (!t->hit)\n");
12038                 break;
12039
12040         case INSTR_JMP_ACTION_HIT:
12041                 fprintf(f,
12042                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12043                         jmp_instr_id);
12044                 break;
12045
12046         case INSTR_JMP_ACTION_MISS:
12047                 fprintf(f,
12048                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12049                         jmp_instr_id);
12050                 break;
12051
12052         case INSTR_JMP_EQ:
12053                 fprintf(f,
12054                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12055                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12056                         jmp_instr_id,
12057                         jmp_instr_id);
12058                 break;
12059
12060         case INSTR_JMP_EQ_MH:
12061                 fprintf(f,
12062                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12063                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12064                         jmp_instr_id,
12065                         jmp_instr_id);
12066                 break;
12067
12068         case INSTR_JMP_EQ_HM:
12069                 fprintf(f,
12070                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12071                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12072                         jmp_instr_id,
12073                         jmp_instr_id);
12074                 break;
12075
12076         case INSTR_JMP_EQ_HH:
12077                 fprintf(f,
12078                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12079                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12080                         jmp_instr_id,
12081                         jmp_instr_id);
12082                 break;
12083
12084         case INSTR_JMP_EQ_I:
12085                 fprintf(f,
12086                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12087                         "pipeline_instructions[%u].jmp.b_val)",
12088                         jmp_instr_id,
12089                         jmp_instr_id);
12090                 break;
12091
12092         case INSTR_JMP_NEQ:
12093                 fprintf(f,
12094                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12095                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12096                         jmp_instr_id,
12097                         jmp_instr_id);
12098                 break;
12099
12100         case INSTR_JMP_NEQ_MH:
12101                 fprintf(f,
12102                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12103                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12104                         jmp_instr_id,
12105                         jmp_instr_id);
12106                 break;
12107
12108         case INSTR_JMP_NEQ_HM:
12109                 fprintf(f,
12110                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12111                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12112                         jmp_instr_id,
12113                         jmp_instr_id);
12114                 break;
12115
12116         case INSTR_JMP_NEQ_HH:
12117                 fprintf(f,
12118                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12119                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12120                         jmp_instr_id,
12121                         jmp_instr_id);
12122                 break;
12123
12124         case INSTR_JMP_NEQ_I:
12125                 fprintf(f,
12126                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12127                         "pipeline_instructions[%u].jmp.b_val)",
12128                         jmp_instr_id,
12129                         jmp_instr_id);
12130                 break;
12131
12132         case INSTR_JMP_LT:
12133                 fprintf(f,
12134                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12135                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12136                         jmp_instr_id,
12137                         jmp_instr_id);
12138                 break;
12139
12140         case INSTR_JMP_LT_MH:
12141                 fprintf(f,
12142                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12143                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12144                         jmp_instr_id,
12145                         jmp_instr_id);
12146                 break;
12147
12148         case INSTR_JMP_LT_HM:
12149                 fprintf(f,
12150                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12151                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12152                         jmp_instr_id,
12153                         jmp_instr_id);
12154                 break;
12155
12156         case INSTR_JMP_LT_HH:
12157                 fprintf(f,
12158                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12159                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12160                         jmp_instr_id,
12161                         jmp_instr_id);
12162                 break;
12163
12164         case INSTR_JMP_LT_MI:
12165                 fprintf(f,
12166                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12167                         "pipeline_instructions[%u].jmp.b_val)",
12168                         jmp_instr_id,
12169                         jmp_instr_id);
12170                 break;
12171
12172         case INSTR_JMP_LT_HI:
12173                 fprintf(f,
12174                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12175                         "pipeline_instructions[%u].jmp.b_val)",
12176                         jmp_instr_id,
12177                         jmp_instr_id);
12178                 break;
12179
12180         case INSTR_JMP_GT:
12181                 fprintf(f,
12182                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12183                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12184                         jmp_instr_id,
12185                         jmp_instr_id);
12186                 break;
12187
12188         case INSTR_JMP_GT_MH:
12189                 fprintf(f,
12190                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12191                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12192                         jmp_instr_id,
12193                         jmp_instr_id);
12194                 break;
12195
12196         case INSTR_JMP_GT_HM:
12197                 fprintf(f,
12198                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12199                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12200                         jmp_instr_id,
12201                         jmp_instr_id);
12202                 break;
12203
12204         case INSTR_JMP_GT_HH:
12205                 fprintf(f,
12206                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12207                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12208                         jmp_instr_id,
12209                         jmp_instr_id);
12210                 break;
12211
12212         case INSTR_JMP_GT_MI:
12213                 fprintf(f,
12214                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12215                         "pipeline_instructions[%u].jmp.b_val)",
12216                         jmp_instr_id,
12217                         jmp_instr_id);
12218                 break;
12219
12220         case INSTR_JMP_GT_HI:
12221                 fprintf(f,
12222                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12223                         "pipeline_instructions[%u].jmp.b_val)",
12224                         jmp_instr_id,
12225                         jmp_instr_id);
12226                 break;
12227
12228         default:
12229                 break;
12230         }
12231
12232         /* Find the instruction group of the jump instruction. */
12233         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12234         if (!jmp_g)
12235                 return -EINVAL;
12236
12237         /* Find the instruction group of the jump destination instruction. */
12238         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12239         if (!data)
12240                 return -EINVAL;
12241
12242         instr_id = data - p->instruction_data;
12243
12244         g = instruction_group_list_group_find(igl, instr_id);
12245         if (!g)
12246                 return -EINVAL;
12247
12248         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12249          * instruction group).
12250          */
12251         if (g->group_id == jmp_g->group_id)
12252                 fprintf(f,
12253                         "\n\t\tgoto %s;\n",
12254                         jmp_data->jmp_label);
12255         else
12256                 fprintf(f,
12257                         " {\n"
12258                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12259                         "\t\treturn;\n"
12260                         "\t}\n\n",
12261                         g->group_id);
12262
12263         return 0;
12264 }
12265
12266 static void
12267 instruction_group_list_codegen(struct instruction_group_list *igl,
12268                                struct rte_swx_pipeline *p,
12269                                FILE *f)
12270 {
12271         struct instruction_group *g;
12272         uint32_t i;
12273         int is_required = 0;
12274
12275         /* Check if code generation is required. */
12276         TAILQ_FOREACH(g, igl, node)
12277                 if (g->first_instr_id < g->last_instr_id)
12278                         is_required = 1;
12279
12280         if (!is_required)
12281                 return;
12282
12283         /* Generate the code for the pipeline instruction array. */
12284         fprintf(f,
12285                 "static const struct instruction pipeline_instructions[] = {\n");
12286
12287         for (i = 0; i < p->n_instructions; i++) {
12288                 struct instruction *instr = &p->instructions[i];
12289                 instruction_export_t func = export_table[instr->type];
12290
12291                 func(instr, f);
12292         }
12293
12294         fprintf(f, "};\n\n");
12295
12296         /* Generate the code for the pipeline functions: one function for each instruction group
12297          * that contains more than one instruction.
12298          */
12299         TAILQ_FOREACH(g, igl, node) {
12300                 struct instruction *last_instr;
12301                 uint32_t j;
12302
12303                 /* Skip if group contains a single instruction. */
12304                 if (g->last_instr_id == g->first_instr_id)
12305                         continue;
12306
12307                 /* Generate new pipeline function. */
12308                 fprintf(f,
12309                         "void\n"
12310                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12311                         "{\n"
12312                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12313                         "\n",
12314                         g->group_id);
12315
12316                 /* Generate the code for each pipeline instruction. */
12317                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12318                         struct instruction *instr = &p->instructions[j];
12319                         struct instruction_data *data = &p->instruction_data[j];
12320
12321                         /* Label, if present. */
12322                         if (data->label[0])
12323                                 fprintf(f, "\n%s : ", data->label);
12324                         else
12325                                 fprintf(f, "\n\t");
12326
12327                         /* TX instruction type. */
12328                         if (instruction_does_tx(instr)) {
12329                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12330                                 continue;
12331                         }
12332
12333                         /* Jump instruction type. */
12334                         if (instruction_is_jmp(instr)) {
12335                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12336                                 continue;
12337                         }
12338
12339                         /* Any other instruction type. */
12340                         fprintf(f,
12341                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12342                                 instr_type_to_func(instr),
12343                                 j);
12344                 }
12345
12346                 /* Finalize the generated pipeline function. For some instructions such as TX,
12347                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12348                  * decided unconditionally and the instruction pointer of the current thread set
12349                  * accordingly; for all the other instructions, the instruction pointer must be
12350                  * incremented now.
12351                  */
12352                 last_instr = &p->instructions[g->last_instr_id];
12353
12354                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12355                         fprintf(f,
12356                                 "thread_ip_inc(p);\n");
12357
12358                 fprintf(f,
12359                         "}\n"
12360                         "\n");
12361         }
12362 }
12363
12364 static uint32_t
12365 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12366 {
12367         struct instruction_group *g;
12368         uint32_t n_custom_instr = 0;
12369
12370         /* Groups with a single instruction: no function is generated for this group, the group
12371          * keeps its current instruction. Groups with more than two instructions: one function and
12372          * the associated custom instruction get generated for each such group.
12373          */
12374         TAILQ_FOREACH(g, igl, node) {
12375                 if (g->first_instr_id == g->last_instr_id)
12376                         continue;
12377
12378                 n_custom_instr++;
12379         }
12380
12381         return n_custom_instr;
12382 }
12383
12384 static int
12385 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12386 {
12387         struct action *a;
12388         FILE *f = NULL;
12389
12390         /* Create the .c file. */
12391         f = fopen("/tmp/pipeline.c", "w");
12392         if (!f)
12393                 return -EIO;
12394
12395         /* Include the .h file. */
12396         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12397
12398         /* Add the code for each action. */
12399         TAILQ_FOREACH(a, &p->actions, node) {
12400                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12401
12402                 action_data_codegen(a, f);
12403
12404                 fprintf(f, "\n");
12405
12406                 action_instr_codegen(a, f);
12407
12408                 fprintf(f, "\n");
12409         }
12410
12411         /* Add the pipeline code. */
12412         instruction_group_list_codegen(igl, p, f);
12413
12414         /* Close the .c file. */
12415         fclose(f);
12416
12417         return 0;
12418 }
12419
12420 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12421 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12422 #endif
12423
12424 static int
12425 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12426 {
12427         struct action *a;
12428         struct instruction_group *g;
12429         char *dir_in, *buffer = NULL;
12430         const char *dir_out;
12431         int status = 0;
12432
12433         /* Get the environment variables. */
12434         dir_in = getenv("RTE_INSTALL_DIR");
12435         if (!dir_in) {
12436                 status = -EINVAL;
12437                 goto free;
12438         }
12439
12440         dir_out = "/tmp";
12441
12442         /* Memory allocation for the command buffer. */
12443         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12444         if (!buffer) {
12445                 status = -ENOMEM;
12446                 goto free;
12447         }
12448
12449         snprintf(buffer,
12450                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12451                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12452                  "-I %s/lib/pipeline "
12453                  "-I %s/lib/eal/include "
12454                  "-I %s/lib/eal/x86/include "
12455                  "-I %s/lib/eal/include/generic "
12456                  "-I %s/lib/meter "
12457                  "-I %s/lib/port "
12458                  "-I %s/lib/table "
12459                  "-I %s/lib/pipeline "
12460                  "-I %s/config "
12461                  "-I %s/build "
12462                  "-I %s/lib/eal/linux/include "
12463                  ">%s/pipeline.log 2>&1 "
12464                  "&& "
12465                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12466                  ">>%s/pipeline.log 2>&1",
12467                  dir_out,
12468                  dir_out,
12469                  dir_in,
12470                  dir_in,
12471                  dir_in,
12472                  dir_in,
12473                  dir_in,
12474                  dir_in,
12475                  dir_in,
12476                  dir_in,
12477                  dir_in,
12478                  dir_in,
12479                  dir_in,
12480                  dir_out,
12481                  dir_out,
12482                  dir_out,
12483                  dir_out);
12484
12485         /* Build the shared object library. */
12486         status = system(buffer);
12487         if (status)
12488                 goto free;
12489
12490         /* Open library. */
12491         snprintf(buffer,
12492                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12493                  "%s/libpipeline.so",
12494                  dir_out);
12495
12496         p->lib = dlopen(buffer, RTLD_LAZY);
12497         if (!p->lib) {
12498                 status = -EIO;
12499                 goto free;
12500         }
12501
12502         /* Get the action function symbols. */
12503         TAILQ_FOREACH(a, &p->actions, node) {
12504                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12505
12506                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12507                 if (!p->action_funcs[a->id]) {
12508                         status = -EINVAL;
12509                         goto free;
12510                 }
12511         }
12512
12513         /* Get the pipeline function symbols. */
12514         TAILQ_FOREACH(g, igl, node) {
12515                 if (g->first_instr_id == g->last_instr_id)
12516                         continue;
12517
12518                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12519
12520                 g->func = dlsym(p->lib, buffer);
12521                 if (!g->func) {
12522                         status = -EINVAL;
12523                         goto free;
12524                 }
12525         }
12526
12527 free:
12528         if (status && p->lib) {
12529                 dlclose(p->lib);
12530                 p->lib = NULL;
12531         }
12532
12533         free(buffer);
12534
12535         return status;
12536 }
12537
12538 static int
12539 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12540                       struct instruction_group_list *igl)
12541 {
12542         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12543
12544         /* Check that enough space is available within the pipeline instruction table to store all
12545          * the custom instructions.
12546          */
12547         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12548                 return -ENOSPC;
12549
12550         return 0;
12551 }
12552
12553 static void
12554 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12555 {
12556         struct instruction_group *g;
12557         uint32_t i;
12558
12559         /* Pipeline table instructions. */
12560         for (i = 0; i < p->n_instructions; i++) {
12561                 struct instruction *instr = &p->instructions[i];
12562
12563                 if (instr->type == INSTR_TABLE)
12564                         instr->type = INSTR_TABLE_AF;
12565
12566                 if (instr->type == INSTR_LEARNER)
12567                         instr->type = INSTR_LEARNER_AF;
12568         }
12569
12570         /* Pipeline custom instructions. */
12571         i = 0;
12572         TAILQ_FOREACH(g, igl, node) {
12573                 struct instruction *instr = &p->instructions[g->first_instr_id];
12574                 uint32_t j;
12575
12576                 if (g->first_instr_id == g->last_instr_id)
12577                         continue;
12578
12579                 /* Install a new custom instruction. */
12580                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12581
12582                 /* First instruction of the group: change its type to the new custom instruction. */
12583                 instr->type = INSTR_CUSTOM_0 + i;
12584
12585                 /* All the subsequent instructions of the group: invalidate. */
12586                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12587                         struct instruction_data *data = &p->instruction_data[j];
12588
12589                         data->invalid = 1;
12590                 }
12591
12592                 i++;
12593         }
12594
12595         /* Remove the invalidated instructions. */
12596         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12597
12598         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12599          * instructions that are the only instruction within their group, so they were left
12600          * unmodified).
12601          */
12602         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12603 }
12604
12605 static int
12606 pipeline_compile(struct rte_swx_pipeline *p)
12607 {
12608         struct instruction_group_list *igl = NULL;
12609         int status = 0;
12610
12611         igl = instruction_group_list_create(p);
12612         if (!igl) {
12613                 status = -ENOMEM;
12614                 goto free;
12615         }
12616
12617         /* Code generation. */
12618         status = pipeline_codegen(p, igl);
12619         if (status)
12620                 goto free;
12621
12622         /* Build and load the shared object library. */
12623         status = pipeline_libload(p, igl);
12624         if (status)
12625                 goto free;
12626
12627         /* Adjust instructions. */
12628         status = pipeline_adjust_check(p, igl);
12629         if (status)
12630                 goto free;
12631
12632         pipeline_adjust(p, igl);
12633
12634 free:
12635         instruction_group_list_free(igl);
12636
12637         return status;
12638 }