lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8 #include <dlfcn.h>
   9
  10 #include <rte_swx_port_ethdev.h>
  11 #include <rte_swx_port_fd.h>
  12 #include <rte_swx_port_ring.h>
  13 #include "rte_swx_port_source_sink.h"
  14
  15 #include "rte_swx_pipeline_internal.h"
  16
  17 #define CHECK(condition, err_code)                                             \
  18 do {                                                                           \
  19         if (!(condition))                                                      \
  20                 return -(err_code);                                            \
  21 } while (0)
  22
  23 #define CHECK_NAME(name, err_code)                                             \
  24         CHECK((name) &&                                                        \
  25               (name)[0] &&                                                     \
  26               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  27               err_code)
  28
  29 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  30         CHECK((instr) &&                                                       \
  31               (instr)[0] &&                                                    \
  32               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  33                RTE_SWX_INSTRUCTION_SIZE),                                      \
  34               err_code)
  35
  36 /*
  37  * Environment.
  38  */
  39 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  40
  41 #include <rte_malloc.h>
  42
  43 static void *
  44 env_malloc(size_t size, size_t alignment, int numa_node)
  45 {
  46         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  47 }
  48
  49 static void
  50 env_free(void *start, size_t size __rte_unused)
  51 {
  52         rte_free(start);
  53 }
  54
  55 #else
  56
  57 #include <numa.h>
  58
  59 static void *
  60 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  61 {
  62         void *start;
  63
  64         if (numa_available() == -1)
  65                 return NULL;
  66
  67         start = numa_alloc_onnode(size, numa_node);
  68         if (!start)
  69                 return NULL;
  70
  71         memset(start, 0, size);
  72         return start;
  73 }
  74
  75 static void
  76 env_free(void *start, size_t size)
  77 {
  78         if (numa_available() == -1)
  79                 return;
  80
  81         numa_free(start, size);
  82 }
  83
  84 #endif
  85
  86 /*
  87  * Struct.
  88  */
  89 static struct struct_type *
  90 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  91 {
  92         struct struct_type *elem;
  93
  94         TAILQ_FOREACH(elem, &p->struct_types, node)
  95                 if (strcmp(elem->name, name) == 0)
  96                         return elem;
  97
  98         return NULL;
  99 }
 100
 101 static struct field *
 102 struct_type_field_find(struct struct_type *st, const char *name)
 103 {
 104         uint32_t i;
 105
 106         for (i = 0; i < st->n_fields; i++) {
 107                 struct field *f = &st->fields[i];
 108
 109                 if (strcmp(f->name, name) == 0)
 110                         return f;
 111         }
 112
 113         return NULL;
 114 }
 115
 116 int
 117 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 118                                       const char *name,
 119                                       struct rte_swx_field_params *fields,
 120                                       uint32_t n_fields,
 121                                       int last_field_has_variable_size)
 122 {
 123         struct struct_type *st;
 124         uint32_t i;
 125
 126         CHECK(p, EINVAL);
 127         CHECK_NAME(name, EINVAL);
 128         CHECK(fields, EINVAL);
 129         CHECK(n_fields, EINVAL);
 130
 131         for (i = 0; i < n_fields; i++) {
 132                 struct rte_swx_field_params *f = &fields[i];
 133                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 134                 uint32_t j;
 135
 136                 CHECK_NAME(f->name, EINVAL);
 137                 CHECK(f->n_bits, EINVAL);
 138                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 139                 CHECK((f->n_bits & 7) == 0, EINVAL);
 140
 141                 for (j = 0; j < i; j++) {
 142                         struct rte_swx_field_params *f_prev = &fields[j];
 143
 144                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 145                 }
 146         }
 147
 148         CHECK(!struct_type_find(p, name), EEXIST);
 149
 150         /* Node allocation. */
 151         st = calloc(1, sizeof(struct struct_type));
 152         CHECK(st, ENOMEM);
 153
 154         st->fields = calloc(n_fields, sizeof(struct field));
 155         if (!st->fields) {
 156                 free(st);
 157                 CHECK(0, ENOMEM);
 158         }
 159
 160         /* Node initialization. */
 161         strcpy(st->name, name);
 162         for (i = 0; i < n_fields; i++) {
 163                 struct field *dst = &st->fields[i];
 164                 struct rte_swx_field_params *src = &fields[i];
 165                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 166
 167                 strcpy(dst->name, src->name);
 168                 dst->n_bits = src->n_bits;
 169                 dst->offset = st->n_bits;
 170                 dst->var_size = var_size;
 171
 172                 st->n_bits += src->n_bits;
 173                 st->n_bits_min += var_size ? 0 : src->n_bits;
 174         }
 175         st->n_fields = n_fields;
 176         st->var_size = last_field_has_variable_size;
 177
 178         /* Node add to tailq. */
 179         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 180
 181         return 0;
 182 }
 183
 184 static int
 185 struct_build(struct rte_swx_pipeline *p)
 186 {
 187         uint32_t i;
 188
 189         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 190                 struct thread *t = &p->threads[i];
 191
 192                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 193                 CHECK(t->structs, ENOMEM);
 194         }
 195
 196         return 0;
 197 }
 198
 199 static void
 200 struct_build_free(struct rte_swx_pipeline *p)
 201 {
 202         uint32_t i;
 203
 204         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 205                 struct thread *t = &p->threads[i];
 206
 207                 free(t->structs);
 208                 t->structs = NULL;
 209         }
 210 }
 211
 212 static void
 213 struct_free(struct rte_swx_pipeline *p)
 214 {
 215         struct_build_free(p);
 216
 217         /* Struct types. */
 218         for ( ; ; ) {
 219                 struct struct_type *elem;
 220
 221                 elem = TAILQ_FIRST(&p->struct_types);
 222                 if (!elem)
 223                         break;
 224
 225                 TAILQ_REMOVE(&p->struct_types, elem, node);
 226                 free(elem->fields);
 227                 free(elem);
 228         }
 229 }
 230
 231 /*
 232  * Input port.
 233  */
 234 static struct port_in_type *
 235 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 236 {
 237         struct port_in_type *elem;
 238
 239         if (!name)
 240                 return NULL;
 241
 242         TAILQ_FOREACH(elem, &p->port_in_types, node)
 243                 if (strcmp(elem->name, name) == 0)
 244                         return elem;
 245
 246         return NULL;
 247 }
 248
 249 int
 250 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 251                                        const char *name,
 252                                        struct rte_swx_port_in_ops *ops)
 253 {
 254         struct port_in_type *elem;
 255
 256         CHECK(p, EINVAL);
 257         CHECK_NAME(name, EINVAL);
 258         CHECK(ops, EINVAL);
 259         CHECK(ops->create, EINVAL);
 260         CHECK(ops->free, EINVAL);
 261         CHECK(ops->pkt_rx, EINVAL);
 262         CHECK(ops->stats_read, EINVAL);
 263
 264         CHECK(!port_in_type_find(p, name), EEXIST);
 265
 266         /* Node allocation. */
 267         elem = calloc(1, sizeof(struct port_in_type));
 268         CHECK(elem, ENOMEM);
 269
 270         /* Node initialization. */
 271         strcpy(elem->name, name);
 272         memcpy(&elem->ops, ops, sizeof(*ops));
 273
 274         /* Node add to tailq. */
 275         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 276
 277         return 0;
 278 }
 279
 280 static struct port_in *
 281 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 282 {
 283         struct port_in *port;
 284
 285         TAILQ_FOREACH(port, &p->ports_in, node)
 286                 if (port->id == port_id)
 287                         return port;
 288
 289         return NULL;
 290 }
 291
 292 int
 293 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 294                                 uint32_t port_id,
 295                                 const char *port_type_name,
 296                                 void *args)
 297 {
 298         struct port_in_type *type = NULL;
 299         struct port_in *port = NULL;
 300         void *obj = NULL;
 301
 302         CHECK(p, EINVAL);
 303
 304         CHECK(!port_in_find(p, port_id), EINVAL);
 305
 306         CHECK_NAME(port_type_name, EINVAL);
 307         type = port_in_type_find(p, port_type_name);
 308         CHECK(type, EINVAL);
 309
 310         obj = type->ops.create(args);
 311         CHECK(obj, ENODEV);
 312
 313         /* Node allocation. */
 314         port = calloc(1, sizeof(struct port_in));
 315         CHECK(port, ENOMEM);
 316
 317         /* Node initialization. */
 318         port->type = type;
 319         port->obj = obj;
 320         port->id = port_id;
 321
 322         /* Node add to tailq. */
 323         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 324         if (p->n_ports_in < port_id + 1)
 325                 p->n_ports_in = port_id + 1;
 326
 327         return 0;
 328 }
 329
 330 static int
 331 port_in_build(struct rte_swx_pipeline *p)
 332 {
 333         struct port_in *port;
 334         uint32_t i;
 335
 336         CHECK(p->n_ports_in, EINVAL);
 337         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 338
 339         for (i = 0; i < p->n_ports_in; i++)
 340                 CHECK(port_in_find(p, i), EINVAL);
 341
 342         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 343         CHECK(p->in, ENOMEM);
 344
 345         TAILQ_FOREACH(port, &p->ports_in, node) {
 346                 struct port_in_runtime *in = &p->in[port->id];
 347
 348                 in->pkt_rx = port->type->ops.pkt_rx;
 349                 in->obj = port->obj;
 350         }
 351
 352         return 0;
 353 }
 354
 355 static void
 356 port_in_build_free(struct rte_swx_pipeline *p)
 357 {
 358         free(p->in);
 359         p->in = NULL;
 360 }
 361
 362 static void
 363 port_in_free(struct rte_swx_pipeline *p)
 364 {
 365         port_in_build_free(p);
 366
 367         /* Input ports. */
 368         for ( ; ; ) {
 369                 struct port_in *port;
 370
 371                 port = TAILQ_FIRST(&p->ports_in);
 372                 if (!port)
 373                         break;
 374
 375                 TAILQ_REMOVE(&p->ports_in, port, node);
 376                 port->type->ops.free(port->obj);
 377                 free(port);
 378         }
 379
 380         /* Input port types. */
 381         for ( ; ; ) {
 382                 struct port_in_type *elem;
 383
 384                 elem = TAILQ_FIRST(&p->port_in_types);
 385                 if (!elem)
 386                         break;
 387
 388                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 389                 free(elem);
 390         }
 391 }
 392
 393 /*
 394  * Output port.
 395  */
 396 static struct port_out_type *
 397 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 398 {
 399         struct port_out_type *elem;
 400
 401         if (!name)
 402                 return NULL;
 403
 404         TAILQ_FOREACH(elem, &p->port_out_types, node)
 405                 if (!strcmp(elem->name, name))
 406                         return elem;
 407
 408         return NULL;
 409 }
 410
 411 int
 412 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 413                                         const char *name,
 414                                         struct rte_swx_port_out_ops *ops)
 415 {
 416         struct port_out_type *elem;
 417
 418         CHECK(p, EINVAL);
 419         CHECK_NAME(name, EINVAL);
 420         CHECK(ops, EINVAL);
 421         CHECK(ops->create, EINVAL);
 422         CHECK(ops->free, EINVAL);
 423         CHECK(ops->pkt_tx, EINVAL);
 424         CHECK(ops->stats_read, EINVAL);
 425
 426         CHECK(!port_out_type_find(p, name), EEXIST);
 427
 428         /* Node allocation. */
 429         elem = calloc(1, sizeof(struct port_out_type));
 430         CHECK(elem, ENOMEM);
 431
 432         /* Node initialization. */
 433         strcpy(elem->name, name);
 434         memcpy(&elem->ops, ops, sizeof(*ops));
 435
 436         /* Node add to tailq. */
 437         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 438
 439         return 0;
 440 }
 441
 442 static struct port_out *
 443 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 444 {
 445         struct port_out *port;
 446
 447         TAILQ_FOREACH(port, &p->ports_out, node)
 448                 if (port->id == port_id)
 449                         return port;
 450
 451         return NULL;
 452 }
 453
 454 int
 455 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 456                                  uint32_t port_id,
 457                                  const char *port_type_name,
 458                                  void *args)
 459 {
 460         struct port_out_type *type = NULL;
 461         struct port_out *port = NULL;
 462         void *obj = NULL;
 463
 464         CHECK(p, EINVAL);
 465
 466         CHECK(!port_out_find(p, port_id), EINVAL);
 467
 468         CHECK_NAME(port_type_name, EINVAL);
 469         type = port_out_type_find(p, port_type_name);
 470         CHECK(type, EINVAL);
 471
 472         obj = type->ops.create(args);
 473         CHECK(obj, ENODEV);
 474
 475         /* Node allocation. */
 476         port = calloc(1, sizeof(struct port_out));
 477         CHECK(port, ENOMEM);
 478
 479         /* Node initialization. */
 480         port->type = type;
 481         port->obj = obj;
 482         port->id = port_id;
 483
 484         /* Node add to tailq. */
 485         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 486         if (p->n_ports_out < port_id + 1)
 487                 p->n_ports_out = port_id + 1;
 488
 489         return 0;
 490 }
 491
 492 static int
 493 port_out_build(struct rte_swx_pipeline *p)
 494 {
 495         struct port_out *port;
 496         uint32_t i;
 497
 498         CHECK(p->n_ports_out, EINVAL);
 499
 500         for (i = 0; i < p->n_ports_out; i++)
 501                 CHECK(port_out_find(p, i), EINVAL);
 502
 503         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 504         CHECK(p->out, ENOMEM);
 505
 506         TAILQ_FOREACH(port, &p->ports_out, node) {
 507                 struct port_out_runtime *out = &p->out[port->id];
 508
 509                 out->pkt_tx = port->type->ops.pkt_tx;
 510                 out->flush = port->type->ops.flush;
 511                 out->obj = port->obj;
 512         }
 513
 514         return 0;
 515 }
 516
 517 static void
 518 port_out_build_free(struct rte_swx_pipeline *p)
 519 {
 520         free(p->out);
 521         p->out = NULL;
 522 }
 523
 524 static void
 525 port_out_free(struct rte_swx_pipeline *p)
 526 {
 527         port_out_build_free(p);
 528
 529         /* Output ports. */
 530         for ( ; ; ) {
 531                 struct port_out *port;
 532
 533                 port = TAILQ_FIRST(&p->ports_out);
 534                 if (!port)
 535                         break;
 536
 537                 TAILQ_REMOVE(&p->ports_out, port, node);
 538                 port->type->ops.free(port->obj);
 539                 free(port);
 540         }
 541
 542         /* Output port types. */
 543         for ( ; ; ) {
 544                 struct port_out_type *elem;
 545
 546                 elem = TAILQ_FIRST(&p->port_out_types);
 547                 if (!elem)
 548                         break;
 549
 550                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 551                 free(elem);
 552         }
 553 }
 554
 555 /*
 556  * Extern object.
 557  */
 558 static struct extern_type *
 559 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 560 {
 561         struct extern_type *elem;
 562
 563         TAILQ_FOREACH(elem, &p->extern_types, node)
 564                 if (strcmp(elem->name, name) == 0)
 565                         return elem;
 566
 567         return NULL;
 568 }
 569
 570 static struct extern_type_member_func *
 571 extern_type_member_func_find(struct extern_type *type, const char *name)
 572 {
 573         struct extern_type_member_func *elem;
 574
 575         TAILQ_FOREACH(elem, &type->funcs, node)
 576                 if (strcmp(elem->name, name) == 0)
 577                         return elem;
 578
 579         return NULL;
 580 }
 581
 582 static struct extern_obj *
 583 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 584 {
 585         struct extern_obj *elem;
 586
 587         TAILQ_FOREACH(elem, &p->extern_objs, node)
 588                 if (strcmp(elem->name, name) == 0)
 589                         return elem;
 590
 591         return NULL;
 592 }
 593
 594 static struct extern_type_member_func *
 595 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 596                              const char *name,
 597                              struct extern_obj **obj)
 598 {
 599         struct extern_obj *object;
 600         struct extern_type_member_func *func;
 601         char *object_name, *func_name;
 602
 603         if (name[0] != 'e' || name[1] != '.')
 604                 return NULL;
 605
 606         object_name = strdup(&name[2]);
 607         if (!object_name)
 608                 return NULL;
 609
 610         func_name = strchr(object_name, '.');
 611         if (!func_name) {
 612                 free(object_name);
 613                 return NULL;
 614         }
 615
 616         *func_name = 0;
 617         func_name++;
 618
 619         object = extern_obj_find(p, object_name);
 620         if (!object) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         func = extern_type_member_func_find(object->type, func_name);
 626         if (!func) {
 627                 free(object_name);
 628                 return NULL;
 629         }
 630
 631         if (obj)
 632                 *obj = object;
 633
 634         free(object_name);
 635         return func;
 636 }
 637
 638 static struct field *
 639 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 640                                const char *name,
 641                                struct extern_obj **object)
 642 {
 643         struct extern_obj *obj;
 644         struct field *f;
 645         char *obj_name, *field_name;
 646
 647         if ((name[0] != 'e') || (name[1] != '.'))
 648                 return NULL;
 649
 650         obj_name = strdup(&name[2]);
 651         if (!obj_name)
 652                 return NULL;
 653
 654         field_name = strchr(obj_name, '.');
 655         if (!field_name) {
 656                 free(obj_name);
 657                 return NULL;
 658         }
 659
 660         *field_name = 0;
 661         field_name++;
 662
 663         obj = extern_obj_find(p, obj_name);
 664         if (!obj) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 670         if (!f) {
 671                 free(obj_name);
 672                 return NULL;
 673         }
 674
 675         if (object)
 676                 *object = obj;
 677
 678         free(obj_name);
 679         return f;
 680 }
 681
 682 int
 683 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 684         const char *name,
 685         const char *mailbox_struct_type_name,
 686         rte_swx_extern_type_constructor_t constructor,
 687         rte_swx_extern_type_destructor_t destructor)
 688 {
 689         struct extern_type *elem;
 690         struct struct_type *mailbox_struct_type;
 691
 692         CHECK(p, EINVAL);
 693
 694         CHECK_NAME(name, EINVAL);
 695         CHECK(!extern_type_find(p, name), EEXIST);
 696
 697         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 698         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 699         CHECK(mailbox_struct_type, EINVAL);
 700         CHECK(!mailbox_struct_type->var_size, EINVAL);
 701
 702         CHECK(constructor, EINVAL);
 703         CHECK(destructor, EINVAL);
 704
 705         /* Node allocation. */
 706         elem = calloc(1, sizeof(struct extern_type));
 707         CHECK(elem, ENOMEM);
 708
 709         /* Node initialization. */
 710         strcpy(elem->name, name);
 711         elem->mailbox_struct_type = mailbox_struct_type;
 712         elem->constructor = constructor;
 713         elem->destructor = destructor;
 714         TAILQ_INIT(&elem->funcs);
 715
 716         /* Node add to tailq. */
 717         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 718
 719         return 0;
 720 }
 721
 722 int
 723 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 724         const char *extern_type_name,
 725         const char *name,
 726         rte_swx_extern_type_member_func_t member_func)
 727 {
 728         struct extern_type *type;
 729         struct extern_type_member_func *type_member;
 730
 731         CHECK(p, EINVAL);
 732
 733         CHECK_NAME(extern_type_name, EINVAL);
 734         type = extern_type_find(p, extern_type_name);
 735         CHECK(type, EINVAL);
 736         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 737
 738         CHECK_NAME(name, EINVAL);
 739         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 740
 741         CHECK(member_func, EINVAL);
 742
 743         /* Node allocation. */
 744         type_member = calloc(1, sizeof(struct extern_type_member_func));
 745         CHECK(type_member, ENOMEM);
 746
 747         /* Node initialization. */
 748         strcpy(type_member->name, name);
 749         type_member->func = member_func;
 750         type_member->id = type->n_funcs;
 751
 752         /* Node add to tailq. */
 753         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 754         type->n_funcs++;
 755
 756         return 0;
 757 }
 758
 759 int
 760 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 761                                       const char *extern_type_name,
 762                                       const char *name,
 763                                       const char *args)
 764 {
 765         struct extern_type *type;
 766         struct extern_obj *obj;
 767         void *obj_handle;
 768
 769         CHECK(p, EINVAL);
 770
 771         CHECK_NAME(extern_type_name, EINVAL);
 772         type = extern_type_find(p, extern_type_name);
 773         CHECK(type, EINVAL);
 774
 775         CHECK_NAME(name, EINVAL);
 776         CHECK(!extern_obj_find(p, name), EEXIST);
 777
 778         /* Node allocation. */
 779         obj = calloc(1, sizeof(struct extern_obj));
 780         CHECK(obj, ENOMEM);
 781
 782         /* Object construction. */
 783         obj_handle = type->constructor(args);
 784         if (!obj_handle) {
 785                 free(obj);
 786                 CHECK(0, ENODEV);
 787         }
 788
 789         /* Node initialization. */
 790         strcpy(obj->name, name);
 791         obj->type = type;
 792         obj->obj = obj_handle;
 793         obj->struct_id = p->n_structs;
 794         obj->id = p->n_extern_objs;
 795
 796         /* Node add to tailq. */
 797         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 798         p->n_extern_objs++;
 799         p->n_structs++;
 800
 801         return 0;
 802 }
 803
 804 static int
 805 extern_obj_build(struct rte_swx_pipeline *p)
 806 {
 807         uint32_t i;
 808
 809         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 810                 struct thread *t = &p->threads[i];
 811                 struct extern_obj *obj;
 812
 813                 t->extern_objs = calloc(p->n_extern_objs,
 814                                         sizeof(struct extern_obj_runtime));
 815                 CHECK(t->extern_objs, ENOMEM);
 816
 817                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 818                         struct extern_obj_runtime *r =
 819                                 &t->extern_objs[obj->id];
 820                         struct extern_type_member_func *func;
 821                         uint32_t mailbox_size =
 822                                 obj->type->mailbox_struct_type->n_bits / 8;
 823
 824                         r->obj = obj->obj;
 825
 826                         r->mailbox = calloc(1, mailbox_size);
 827                         CHECK(r->mailbox, ENOMEM);
 828
 829                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 830                                 r->funcs[func->id] = func->func;
 831
 832                         t->structs[obj->struct_id] = r->mailbox;
 833                 }
 834         }
 835
 836         return 0;
 837 }
 838
 839 static void
 840 extern_obj_build_free(struct rte_swx_pipeline *p)
 841 {
 842         uint32_t i;
 843
 844         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 845                 struct thread *t = &p->threads[i];
 846                 uint32_t j;
 847
 848                 if (!t->extern_objs)
 849                         continue;
 850
 851                 for (j = 0; j < p->n_extern_objs; j++) {
 852                         struct extern_obj_runtime *r = &t->extern_objs[j];
 853
 854                         free(r->mailbox);
 855                 }
 856
 857                 free(t->extern_objs);
 858                 t->extern_objs = NULL;
 859         }
 860 }
 861
 862 static void
 863 extern_obj_free(struct rte_swx_pipeline *p)
 864 {
 865         extern_obj_build_free(p);
 866
 867         /* Extern objects. */
 868         for ( ; ; ) {
 869                 struct extern_obj *elem;
 870
 871                 elem = TAILQ_FIRST(&p->extern_objs);
 872                 if (!elem)
 873                         break;
 874
 875                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 876                 if (elem->obj)
 877                         elem->type->destructor(elem->obj);
 878                 free(elem);
 879         }
 880
 881         /* Extern types. */
 882         for ( ; ; ) {
 883                 struct extern_type *elem;
 884
 885                 elem = TAILQ_FIRST(&p->extern_types);
 886                 if (!elem)
 887                         break;
 888
 889                 TAILQ_REMOVE(&p->extern_types, elem, node);
 890
 891                 for ( ; ; ) {
 892                         struct extern_type_member_func *func;
 893
 894                         func = TAILQ_FIRST(&elem->funcs);
 895                         if (!func)
 896                                 break;
 897
 898                         TAILQ_REMOVE(&elem->funcs, func, node);
 899                         free(func);
 900                 }
 901
 902                 free(elem);
 903         }
 904 }
 905
 906 /*
 907  * Extern function.
 908  */
 909 static struct extern_func *
 910 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 911 {
 912         struct extern_func *elem;
 913
 914         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 915                 if (strcmp(elem->name, name) == 0)
 916                         return elem;
 917
 918         return NULL;
 919 }
 920
 921 static struct extern_func *
 922 extern_func_parse(struct rte_swx_pipeline *p,
 923                   const char *name)
 924 {
 925         if (name[0] != 'f' || name[1] != '.')
 926                 return NULL;
 927
 928         return extern_func_find(p, &name[2]);
 929 }
 930
 931 static struct field *
 932 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 933                                 const char *name,
 934                                 struct extern_func **function)
 935 {
 936         struct extern_func *func;
 937         struct field *f;
 938         char *func_name, *field_name;
 939
 940         if ((name[0] != 'f') || (name[1] != '.'))
 941                 return NULL;
 942
 943         func_name = strdup(&name[2]);
 944         if (!func_name)
 945                 return NULL;
 946
 947         field_name = strchr(func_name, '.');
 948         if (!field_name) {
 949                 free(func_name);
 950                 return NULL;
 951         }
 952
 953         *field_name = 0;
 954         field_name++;
 955
 956         func = extern_func_find(p, func_name);
 957         if (!func) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 963         if (!f) {
 964                 free(func_name);
 965                 return NULL;
 966         }
 967
 968         if (function)
 969                 *function = func;
 970
 971         free(func_name);
 972         return f;
 973 }
 974
 975 int
 976 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 977                                       const char *name,
 978                                       const char *mailbox_struct_type_name,
 979                                       rte_swx_extern_func_t func)
 980 {
 981         struct extern_func *f;
 982         struct struct_type *mailbox_struct_type;
 983
 984         CHECK(p, EINVAL);
 985
 986         CHECK_NAME(name, EINVAL);
 987         CHECK(!extern_func_find(p, name), EEXIST);
 988
 989         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 990         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 991         CHECK(mailbox_struct_type, EINVAL);
 992         CHECK(!mailbox_struct_type->var_size, EINVAL);
 993
 994         CHECK(func, EINVAL);
 995
 996         /* Node allocation. */
 997         f = calloc(1, sizeof(struct extern_func));
 998         CHECK(func, ENOMEM);
 999
1000         /* Node initialization. */
1001         strcpy(f->name, name);
1002         f->mailbox_struct_type = mailbox_struct_type;
1003         f->func = func;
1004         f->struct_id = p->n_structs;
1005         f->id = p->n_extern_funcs;
1006
1007         /* Node add to tailq. */
1008         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1009         p->n_extern_funcs++;
1010         p->n_structs++;
1011
1012         return 0;
1013 }
1014
1015 static int
1016 extern_func_build(struct rte_swx_pipeline *p)
1017 {
1018         uint32_t i;
1019
1020         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1021                 struct thread *t = &p->threads[i];
1022                 struct extern_func *func;
1023
1024                 /* Memory allocation. */
1025                 t->extern_funcs = calloc(p->n_extern_funcs,
1026                                          sizeof(struct extern_func_runtime));
1027                 CHECK(t->extern_funcs, ENOMEM);
1028
1029                 /* Extern function. */
1030                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1031                         struct extern_func_runtime *r =
1032                                 &t->extern_funcs[func->id];
1033                         uint32_t mailbox_size =
1034                                 func->mailbox_struct_type->n_bits / 8;
1035
1036                         r->func = func->func;
1037
1038                         r->mailbox = calloc(1, mailbox_size);
1039                         CHECK(r->mailbox, ENOMEM);
1040
1041                         t->structs[func->struct_id] = r->mailbox;
1042                 }
1043         }
1044
1045         return 0;
1046 }
1047
1048 static void
1049 extern_func_build_free(struct rte_swx_pipeline *p)
1050 {
1051         uint32_t i;
1052
1053         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1054                 struct thread *t = &p->threads[i];
1055                 uint32_t j;
1056
1057                 if (!t->extern_funcs)
1058                         continue;
1059
1060                 for (j = 0; j < p->n_extern_funcs; j++) {
1061                         struct extern_func_runtime *r = &t->extern_funcs[j];
1062
1063                         free(r->mailbox);
1064                 }
1065
1066                 free(t->extern_funcs);
1067                 t->extern_funcs = NULL;
1068         }
1069 }
1070
1071 static void
1072 extern_func_free(struct rte_swx_pipeline *p)
1073 {
1074         extern_func_build_free(p);
1075
1076         for ( ; ; ) {
1077                 struct extern_func *elem;
1078
1079                 elem = TAILQ_FIRST(&p->extern_funcs);
1080                 if (!elem)
1081                         break;
1082
1083                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1084                 free(elem);
1085         }
1086 }
1087
1088 /*
1089  * Header.
1090  */
1091 static struct header *
1092 header_find(struct rte_swx_pipeline *p, const char *name)
1093 {
1094         struct header *elem;
1095
1096         TAILQ_FOREACH(elem, &p->headers, node)
1097                 if (strcmp(elem->name, name) == 0)
1098                         return elem;
1099
1100         return NULL;
1101 }
1102
1103 static struct header *
1104 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1105 {
1106         struct header *elem;
1107
1108         TAILQ_FOREACH(elem, &p->headers, node)
1109                 if (elem->struct_id == struct_id)
1110                         return elem;
1111
1112         return NULL;
1113 }
1114
1115 static struct header *
1116 header_parse(struct rte_swx_pipeline *p,
1117              const char *name)
1118 {
1119         if (name[0] != 'h' || name[1] != '.')
1120                 return NULL;
1121
1122         return header_find(p, &name[2]);
1123 }
1124
1125 static struct field *
1126 header_field_parse(struct rte_swx_pipeline *p,
1127                    const char *name,
1128                    struct header **header)
1129 {
1130         struct header *h;
1131         struct field *f;
1132         char *header_name, *field_name;
1133
1134         if ((name[0] != 'h') || (name[1] != '.'))
1135                 return NULL;
1136
1137         header_name = strdup(&name[2]);
1138         if (!header_name)
1139                 return NULL;
1140
1141         field_name = strchr(header_name, '.');
1142         if (!field_name) {
1143                 free(header_name);
1144                 return NULL;
1145         }
1146
1147         *field_name = 0;
1148         field_name++;
1149
1150         h = header_find(p, header_name);
1151         if (!h) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         f = struct_type_field_find(h->st, field_name);
1157         if (!f) {
1158                 free(header_name);
1159                 return NULL;
1160         }
1161
1162         if (header)
1163                 *header = h;
1164
1165         free(header_name);
1166         return f;
1167 }
1168
1169 int
1170 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1171                                         const char *name,
1172                                         const char *struct_type_name)
1173 {
1174         struct struct_type *st;
1175         struct header *h;
1176         size_t n_headers_max;
1177
1178         CHECK(p, EINVAL);
1179         CHECK_NAME(name, EINVAL);
1180         CHECK_NAME(struct_type_name, EINVAL);
1181
1182         CHECK(!header_find(p, name), EEXIST);
1183
1184         st = struct_type_find(p, struct_type_name);
1185         CHECK(st, EINVAL);
1186
1187         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1188         CHECK(p->n_headers < n_headers_max, ENOSPC);
1189
1190         /* Node allocation. */
1191         h = calloc(1, sizeof(struct header));
1192         CHECK(h, ENOMEM);
1193
1194         /* Node initialization. */
1195         strcpy(h->name, name);
1196         h->st = st;
1197         h->struct_id = p->n_structs;
1198         h->id = p->n_headers;
1199
1200         /* Node add to tailq. */
1201         TAILQ_INSERT_TAIL(&p->headers, h, node);
1202         p->n_headers++;
1203         p->n_structs++;
1204
1205         return 0;
1206 }
1207
1208 static int
1209 header_build(struct rte_swx_pipeline *p)
1210 {
1211         struct header *h;
1212         uint32_t n_bytes = 0, i;
1213
1214         TAILQ_FOREACH(h, &p->headers, node) {
1215                 n_bytes += h->st->n_bits / 8;
1216         }
1217
1218         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1219                 struct thread *t = &p->threads[i];
1220                 uint32_t offset = 0;
1221
1222                 t->headers = calloc(p->n_headers,
1223                                     sizeof(struct header_runtime));
1224                 CHECK(t->headers, ENOMEM);
1225
1226                 t->headers_out = calloc(p->n_headers,
1227                                         sizeof(struct header_out_runtime));
1228                 CHECK(t->headers_out, ENOMEM);
1229
1230                 t->header_storage = calloc(1, n_bytes);
1231                 CHECK(t->header_storage, ENOMEM);
1232
1233                 t->header_out_storage = calloc(1, n_bytes);
1234                 CHECK(t->header_out_storage, ENOMEM);
1235
1236                 TAILQ_FOREACH(h, &p->headers, node) {
1237                         uint8_t *header_storage;
1238                         uint32_t n_bytes =  h->st->n_bits / 8;
1239
1240                         header_storage = &t->header_storage[offset];
1241                         offset += n_bytes;
1242
1243                         t->headers[h->id].ptr0 = header_storage;
1244                         t->headers[h->id].n_bytes = n_bytes;
1245
1246                         t->structs[h->struct_id] = header_storage;
1247                 }
1248         }
1249
1250         return 0;
1251 }
1252
1253 static void
1254 header_build_free(struct rte_swx_pipeline *p)
1255 {
1256         uint32_t i;
1257
1258         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1259                 struct thread *t = &p->threads[i];
1260
1261                 free(t->headers_out);
1262                 t->headers_out = NULL;
1263
1264                 free(t->headers);
1265                 t->headers = NULL;
1266
1267                 free(t->header_out_storage);
1268                 t->header_out_storage = NULL;
1269
1270                 free(t->header_storage);
1271                 t->header_storage = NULL;
1272         }
1273 }
1274
1275 static void
1276 header_free(struct rte_swx_pipeline *p)
1277 {
1278         header_build_free(p);
1279
1280         for ( ; ; ) {
1281                 struct header *elem;
1282
1283                 elem = TAILQ_FIRST(&p->headers);
1284                 if (!elem)
1285                         break;
1286
1287                 TAILQ_REMOVE(&p->headers, elem, node);
1288                 free(elem);
1289         }
1290 }
1291
1292 /*
1293  * Meta-data.
1294  */
1295 static struct field *
1296 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1297 {
1298         if (!p->metadata_st)
1299                 return NULL;
1300
1301         if (name[0] != 'm' || name[1] != '.')
1302                 return NULL;
1303
1304         return struct_type_field_find(p->metadata_st, &name[2]);
1305 }
1306
1307 int
1308 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1309                                           const char *struct_type_name)
1310 {
1311         struct struct_type *st = NULL;
1312
1313         CHECK(p, EINVAL);
1314
1315         CHECK_NAME(struct_type_name, EINVAL);
1316         st  = struct_type_find(p, struct_type_name);
1317         CHECK(st, EINVAL);
1318         CHECK(!st->var_size, EINVAL);
1319         CHECK(!p->metadata_st, EINVAL);
1320
1321         p->metadata_st = st;
1322         p->metadata_struct_id = p->n_structs;
1323
1324         p->n_structs++;
1325
1326         return 0;
1327 }
1328
1329 static int
1330 metadata_build(struct rte_swx_pipeline *p)
1331 {
1332         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1333         uint32_t i;
1334
1335         /* Thread-level initialization. */
1336         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1337                 struct thread *t = &p->threads[i];
1338                 uint8_t *metadata;
1339
1340                 metadata = calloc(1, n_bytes);
1341                 CHECK(metadata, ENOMEM);
1342
1343                 t->metadata = metadata;
1344                 t->structs[p->metadata_struct_id] = metadata;
1345         }
1346
1347         return 0;
1348 }
1349
1350 static void
1351 metadata_build_free(struct rte_swx_pipeline *p)
1352 {
1353         uint32_t i;
1354
1355         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1356                 struct thread *t = &p->threads[i];
1357
1358                 free(t->metadata);
1359                 t->metadata = NULL;
1360         }
1361 }
1362
1363 static void
1364 metadata_free(struct rte_swx_pipeline *p)
1365 {
1366         metadata_build_free(p);
1367 }
1368
1369 /*
1370  * Instruction.
1371  */
1372 static int
1373 instruction_is_tx(enum instruction_type type)
1374 {
1375         switch (type) {
1376         case INSTR_TX:
1377         case INSTR_TX_I:
1378         case INSTR_DROP:
1379                 return 1;
1380
1381         default:
1382                 return 0;
1383         }
1384 }
1385
1386 static int
1387 instruction_does_tx(struct instruction *instr)
1388 {
1389         switch (instr->type) {
1390         case INSTR_TX:
1391         case INSTR_TX_I:
1392         case INSTR_DROP:
1393         case INSTR_HDR_EMIT_TX:
1394         case INSTR_HDR_EMIT2_TX:
1395         case INSTR_HDR_EMIT3_TX:
1396         case INSTR_HDR_EMIT4_TX:
1397         case INSTR_HDR_EMIT5_TX:
1398         case INSTR_HDR_EMIT6_TX:
1399         case INSTR_HDR_EMIT7_TX:
1400         case INSTR_HDR_EMIT8_TX:
1401                 return 1;
1402         default:
1403                 return 0;
1404         }
1405 }
1406
1407 static int
1408 instruction_is_jmp(struct instruction *instr)
1409 {
1410         switch (instr->type) {
1411         case INSTR_JMP:
1412         case INSTR_JMP_VALID:
1413         case INSTR_JMP_INVALID:
1414         case INSTR_JMP_HIT:
1415         case INSTR_JMP_MISS:
1416         case INSTR_JMP_ACTION_HIT:
1417         case INSTR_JMP_ACTION_MISS:
1418         case INSTR_JMP_EQ:
1419         case INSTR_JMP_EQ_MH:
1420         case INSTR_JMP_EQ_HM:
1421         case INSTR_JMP_EQ_HH:
1422         case INSTR_JMP_EQ_I:
1423         case INSTR_JMP_NEQ:
1424         case INSTR_JMP_NEQ_MH:
1425         case INSTR_JMP_NEQ_HM:
1426         case INSTR_JMP_NEQ_HH:
1427         case INSTR_JMP_NEQ_I:
1428         case INSTR_JMP_LT:
1429         case INSTR_JMP_LT_MH:
1430         case INSTR_JMP_LT_HM:
1431         case INSTR_JMP_LT_HH:
1432         case INSTR_JMP_LT_MI:
1433         case INSTR_JMP_LT_HI:
1434         case INSTR_JMP_GT:
1435         case INSTR_JMP_GT_MH:
1436         case INSTR_JMP_GT_HM:
1437         case INSTR_JMP_GT_HH:
1438         case INSTR_JMP_GT_MI:
1439         case INSTR_JMP_GT_HI:
1440                 return 1;
1441
1442         default:
1443                 return 0;
1444         }
1445 }
1446
1447 static int
1448 instruction_does_thread_yield(struct instruction *instr)
1449 {
1450         switch (instr->type) {
1451         case INSTR_RX:
1452         case INSTR_TABLE:
1453         case INSTR_TABLE_AF:
1454         case INSTR_SELECTOR:
1455         case INSTR_LEARNER:
1456         case INSTR_LEARNER_AF:
1457         case INSTR_EXTERN_OBJ:
1458         case INSTR_EXTERN_FUNC:
1459                 return 1;
1460         default:
1461                 return 0;
1462         }
1463 }
1464
1465 static struct field *
1466 action_field_parse(struct action *action, const char *name);
1467
1468 static struct field *
1469 struct_field_parse(struct rte_swx_pipeline *p,
1470                    struct action *action,
1471                    const char *name,
1472                    uint32_t *struct_id)
1473 {
1474         struct field *f;
1475
1476         switch (name[0]) {
1477         case 'h':
1478         {
1479                 struct header *header;
1480
1481                 f = header_field_parse(p, name, &header);
1482                 if (!f)
1483                         return NULL;
1484
1485                 *struct_id = header->struct_id;
1486                 return f;
1487         }
1488
1489         case 'm':
1490         {
1491                 f = metadata_field_parse(p, name);
1492                 if (!f)
1493                         return NULL;
1494
1495                 *struct_id = p->metadata_struct_id;
1496                 return f;
1497         }
1498
1499         case 't':
1500         {
1501                 if (!action)
1502                         return NULL;
1503
1504                 f = action_field_parse(action, name);
1505                 if (!f)
1506                         return NULL;
1507
1508                 *struct_id = 0;
1509                 return f;
1510         }
1511
1512         case 'e':
1513         {
1514                 struct extern_obj *obj;
1515
1516                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1517                 if (!f)
1518                         return NULL;
1519
1520                 *struct_id = obj->struct_id;
1521                 return f;
1522         }
1523
1524         case 'f':
1525         {
1526                 struct extern_func *func;
1527
1528                 f = extern_func_mailbox_field_parse(p, name, &func);
1529                 if (!f)
1530                         return NULL;
1531
1532                 *struct_id = func->struct_id;
1533                 return f;
1534         }
1535
1536         default:
1537                 return NULL;
1538         }
1539 }
1540
1541 /*
1542  * rx.
1543  */
1544 static int
1545 instr_rx_translate(struct rte_swx_pipeline *p,
1546                    struct action *action,
1547                    char **tokens,
1548                    int n_tokens,
1549                    struct instruction *instr,
1550                    struct instruction_data *data __rte_unused)
1551 {
1552         struct field *f;
1553
1554         CHECK(!action, EINVAL);
1555         CHECK(n_tokens == 2, EINVAL);
1556
1557         f = metadata_field_parse(p, tokens[1]);
1558         CHECK(f, EINVAL);
1559
1560         instr->type = INSTR_RX;
1561         instr->io.io.offset = f->offset / 8;
1562         instr->io.io.n_bits = f->n_bits;
1563         return 0;
1564 }
1565
1566 /*
1567  * tx.
1568  */
1569 static int
1570 instr_tx_translate(struct rte_swx_pipeline *p,
1571                    struct action *action __rte_unused,
1572                    char **tokens,
1573                    int n_tokens,
1574                    struct instruction *instr,
1575                    struct instruction_data *data __rte_unused)
1576 {
1577         char *port = tokens[1];
1578         struct field *f;
1579         uint32_t port_val;
1580
1581         CHECK(n_tokens == 2, EINVAL);
1582
1583         f = metadata_field_parse(p, port);
1584         if (f) {
1585                 instr->type = INSTR_TX;
1586                 instr->io.io.offset = f->offset / 8;
1587                 instr->io.io.n_bits = f->n_bits;
1588                 return 0;
1589         }
1590
1591         /* TX_I. */
1592         port_val = strtoul(port, &port, 0);
1593         CHECK(!port[0], EINVAL);
1594
1595         instr->type = INSTR_TX_I;
1596         instr->io.io.val = port_val;
1597         return 0;
1598 }
1599
1600 static int
1601 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1602                      struct action *action __rte_unused,
1603                      char **tokens __rte_unused,
1604                      int n_tokens,
1605                      struct instruction *instr,
1606                      struct instruction_data *data __rte_unused)
1607 {
1608         CHECK(n_tokens == 1, EINVAL);
1609
1610         /* DROP. */
1611         instr->type = INSTR_DROP;
1612         return 0;
1613 }
1614
1615 static inline void
1616 instr_tx_exec(struct rte_swx_pipeline *p)
1617 {
1618         struct thread *t = &p->threads[p->thread_id];
1619         struct instruction *ip = t->ip;
1620
1621         __instr_tx_exec(p, t, ip);
1622
1623         /* Thread. */
1624         thread_ip_reset(p, t);
1625         instr_rx_exec(p);
1626 }
1627
1628 static inline void
1629 instr_tx_i_exec(struct rte_swx_pipeline *p)
1630 {
1631         struct thread *t = &p->threads[p->thread_id];
1632         struct instruction *ip = t->ip;
1633
1634         __instr_tx_i_exec(p, t, ip);
1635
1636         /* Thread. */
1637         thread_ip_reset(p, t);
1638         instr_rx_exec(p);
1639 }
1640
1641 static inline void
1642 instr_drop_exec(struct rte_swx_pipeline *p)
1643 {
1644         struct thread *t = &p->threads[p->thread_id];
1645         struct instruction *ip = t->ip;
1646
1647         __instr_drop_exec(p, t, ip);
1648
1649         /* Thread. */
1650         thread_ip_reset(p, t);
1651         instr_rx_exec(p);
1652 }
1653
1654 /*
1655  * extract.
1656  */
1657 static int
1658 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1659                             struct action *action,
1660                             char **tokens,
1661                             int n_tokens,
1662                             struct instruction *instr,
1663                             struct instruction_data *data __rte_unused)
1664 {
1665         struct header *h;
1666
1667         CHECK(!action, EINVAL);
1668         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1669
1670         h = header_parse(p, tokens[1]);
1671         CHECK(h, EINVAL);
1672
1673         if (n_tokens == 2) {
1674                 CHECK(!h->st->var_size, EINVAL);
1675
1676                 instr->type = INSTR_HDR_EXTRACT;
1677                 instr->io.hdr.header_id[0] = h->id;
1678                 instr->io.hdr.struct_id[0] = h->struct_id;
1679                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1680         } else {
1681                 struct field *mf;
1682
1683                 CHECK(h->st->var_size, EINVAL);
1684
1685                 mf = metadata_field_parse(p, tokens[2]);
1686                 CHECK(mf, EINVAL);
1687                 CHECK(!mf->var_size, EINVAL);
1688
1689                 instr->type = INSTR_HDR_EXTRACT_M;
1690                 instr->io.io.offset = mf->offset / 8;
1691                 instr->io.io.n_bits = mf->n_bits;
1692                 instr->io.hdr.header_id[0] = h->id;
1693                 instr->io.hdr.struct_id[0] = h->struct_id;
1694                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1695         }
1696
1697         return 0;
1698 }
1699
1700 static int
1701 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1702                               struct action *action,
1703                               char **tokens,
1704                               int n_tokens,
1705                               struct instruction *instr,
1706                               struct instruction_data *data __rte_unused)
1707 {
1708         struct header *h;
1709
1710         CHECK(!action, EINVAL);
1711         CHECK(n_tokens == 2, EINVAL);
1712
1713         h = header_parse(p, tokens[1]);
1714         CHECK(h, EINVAL);
1715         CHECK(!h->st->var_size, EINVAL);
1716
1717         instr->type = INSTR_HDR_LOOKAHEAD;
1718         instr->io.hdr.header_id[0] = h->id;
1719         instr->io.hdr.struct_id[0] = h->struct_id;
1720         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1721
1722         return 0;
1723 }
1724
1725 static inline void
1726 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1727 {
1728         struct thread *t = &p->threads[p->thread_id];
1729         struct instruction *ip = t->ip;
1730
1731         __instr_hdr_extract_exec(p, t, ip);
1732
1733         /* Thread. */
1734         thread_ip_inc(p);
1735 }
1736
1737 static inline void
1738 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1739 {
1740         struct thread *t = &p->threads[p->thread_id];
1741         struct instruction *ip = t->ip;
1742
1743         __instr_hdr_extract2_exec(p, t, ip);
1744
1745         /* Thread. */
1746         thread_ip_inc(p);
1747 }
1748
1749 static inline void
1750 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1751 {
1752         struct thread *t = &p->threads[p->thread_id];
1753         struct instruction *ip = t->ip;
1754
1755         __instr_hdr_extract3_exec(p, t, ip);
1756
1757         /* Thread. */
1758         thread_ip_inc(p);
1759 }
1760
1761 static inline void
1762 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1763 {
1764         struct thread *t = &p->threads[p->thread_id];
1765         struct instruction *ip = t->ip;
1766
1767         __instr_hdr_extract4_exec(p, t, ip);
1768
1769         /* Thread. */
1770         thread_ip_inc(p);
1771 }
1772
1773 static inline void
1774 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1775 {
1776         struct thread *t = &p->threads[p->thread_id];
1777         struct instruction *ip = t->ip;
1778
1779         __instr_hdr_extract5_exec(p, t, ip);
1780
1781         /* Thread. */
1782         thread_ip_inc(p);
1783 }
1784
1785 static inline void
1786 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1787 {
1788         struct thread *t = &p->threads[p->thread_id];
1789         struct instruction *ip = t->ip;
1790
1791         __instr_hdr_extract6_exec(p, t, ip);
1792
1793         /* Thread. */
1794         thread_ip_inc(p);
1795 }
1796
1797 static inline void
1798 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1799 {
1800         struct thread *t = &p->threads[p->thread_id];
1801         struct instruction *ip = t->ip;
1802
1803         __instr_hdr_extract7_exec(p, t, ip);
1804
1805         /* Thread. */
1806         thread_ip_inc(p);
1807 }
1808
1809 static inline void
1810 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1811 {
1812         struct thread *t = &p->threads[p->thread_id];
1813         struct instruction *ip = t->ip;
1814
1815         __instr_hdr_extract8_exec(p, t, ip);
1816
1817         /* Thread. */
1818         thread_ip_inc(p);
1819 }
1820
1821 static inline void
1822 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1823 {
1824         struct thread *t = &p->threads[p->thread_id];
1825         struct instruction *ip = t->ip;
1826
1827         __instr_hdr_extract_m_exec(p, t, ip);
1828
1829         /* Thread. */
1830         thread_ip_inc(p);
1831 }
1832
1833 static inline void
1834 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1835 {
1836         struct thread *t = &p->threads[p->thread_id];
1837         struct instruction *ip = t->ip;
1838
1839         __instr_hdr_lookahead_exec(p, t, ip);
1840
1841         /* Thread. */
1842         thread_ip_inc(p);
1843 }
1844
1845 /*
1846  * emit.
1847  */
1848 static int
1849 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1850                          struct action *action __rte_unused,
1851                          char **tokens,
1852                          int n_tokens,
1853                          struct instruction *instr,
1854                          struct instruction_data *data __rte_unused)
1855 {
1856         struct header *h;
1857
1858         CHECK(n_tokens == 2, EINVAL);
1859
1860         h = header_parse(p, tokens[1]);
1861         CHECK(h, EINVAL);
1862
1863         instr->type = INSTR_HDR_EMIT;
1864         instr->io.hdr.header_id[0] = h->id;
1865         instr->io.hdr.struct_id[0] = h->struct_id;
1866         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1867         return 0;
1868 }
1869
1870 static inline void
1871 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1872 {
1873         struct thread *t = &p->threads[p->thread_id];
1874         struct instruction *ip = t->ip;
1875
1876         __instr_hdr_emit_exec(p, t, ip);
1877
1878         /* Thread. */
1879         thread_ip_inc(p);
1880 }
1881
1882 static inline void
1883 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1884 {
1885         struct thread *t = &p->threads[p->thread_id];
1886         struct instruction *ip = t->ip;
1887
1888         __instr_hdr_emit_tx_exec(p, t, ip);
1889
1890         /* Thread. */
1891         thread_ip_reset(p, t);
1892         instr_rx_exec(p);
1893 }
1894
1895 static inline void
1896 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1897 {
1898         struct thread *t = &p->threads[p->thread_id];
1899         struct instruction *ip = t->ip;
1900
1901         __instr_hdr_emit2_tx_exec(p, t, ip);
1902
1903         /* Thread. */
1904         thread_ip_reset(p, t);
1905         instr_rx_exec(p);
1906 }
1907
1908 static inline void
1909 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1910 {
1911         struct thread *t = &p->threads[p->thread_id];
1912         struct instruction *ip = t->ip;
1913
1914         __instr_hdr_emit3_tx_exec(p, t, ip);
1915
1916         /* Thread. */
1917         thread_ip_reset(p, t);
1918         instr_rx_exec(p);
1919 }
1920
1921 static inline void
1922 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1923 {
1924         struct thread *t = &p->threads[p->thread_id];
1925         struct instruction *ip = t->ip;
1926
1927         __instr_hdr_emit4_tx_exec(p, t, ip);
1928
1929         /* Thread. */
1930         thread_ip_reset(p, t);
1931         instr_rx_exec(p);
1932 }
1933
1934 static inline void
1935 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1936 {
1937         struct thread *t = &p->threads[p->thread_id];
1938         struct instruction *ip = t->ip;
1939
1940         __instr_hdr_emit5_tx_exec(p, t, ip);
1941
1942         /* Thread. */
1943         thread_ip_reset(p, t);
1944         instr_rx_exec(p);
1945 }
1946
1947 static inline void
1948 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1949 {
1950         struct thread *t = &p->threads[p->thread_id];
1951         struct instruction *ip = t->ip;
1952
1953         __instr_hdr_emit6_tx_exec(p, t, ip);
1954
1955         /* Thread. */
1956         thread_ip_reset(p, t);
1957         instr_rx_exec(p);
1958 }
1959
1960 static inline void
1961 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1962 {
1963         struct thread *t = &p->threads[p->thread_id];
1964         struct instruction *ip = t->ip;
1965
1966         __instr_hdr_emit7_tx_exec(p, t, ip);
1967
1968         /* Thread. */
1969         thread_ip_reset(p, t);
1970         instr_rx_exec(p);
1971 }
1972
1973 static inline void
1974 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1975 {
1976         struct thread *t = &p->threads[p->thread_id];
1977         struct instruction *ip = t->ip;
1978
1979         __instr_hdr_emit8_tx_exec(p, t, ip);
1980
1981         /* Thread. */
1982         thread_ip_reset(p, t);
1983         instr_rx_exec(p);
1984 }
1985
1986 /*
1987  * validate.
1988  */
1989 static int
1990 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1991                              struct action *action __rte_unused,
1992                              char **tokens,
1993                              int n_tokens,
1994                              struct instruction *instr,
1995                              struct instruction_data *data __rte_unused)
1996 {
1997         struct header *h;
1998
1999         CHECK(n_tokens == 2, EINVAL);
2000
2001         h = header_parse(p, tokens[1]);
2002         CHECK(h, EINVAL);
2003
2004         instr->type = INSTR_HDR_VALIDATE;
2005         instr->valid.header_id = h->id;
2006         return 0;
2007 }
2008
2009 static inline void
2010 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2011 {
2012         struct thread *t = &p->threads[p->thread_id];
2013         struct instruction *ip = t->ip;
2014
2015         __instr_hdr_validate_exec(p, t, ip);
2016
2017         /* Thread. */
2018         thread_ip_inc(p);
2019 }
2020
2021 /*
2022  * invalidate.
2023  */
2024 static int
2025 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2026                                struct action *action __rte_unused,
2027                                char **tokens,
2028                                int n_tokens,
2029                                struct instruction *instr,
2030                                struct instruction_data *data __rte_unused)
2031 {
2032         struct header *h;
2033
2034         CHECK(n_tokens == 2, EINVAL);
2035
2036         h = header_parse(p, tokens[1]);
2037         CHECK(h, EINVAL);
2038
2039         instr->type = INSTR_HDR_INVALIDATE;
2040         instr->valid.header_id = h->id;
2041         return 0;
2042 }
2043
2044 static inline void
2045 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2046 {
2047         struct thread *t = &p->threads[p->thread_id];
2048         struct instruction *ip = t->ip;
2049
2050         __instr_hdr_invalidate_exec(p, t, ip);
2051
2052         /* Thread. */
2053         thread_ip_inc(p);
2054 }
2055
2056 /*
2057  * table.
2058  */
2059 static struct table *
2060 table_find(struct rte_swx_pipeline *p, const char *name);
2061
2062 static struct selector *
2063 selector_find(struct rte_swx_pipeline *p, const char *name);
2064
2065 static struct learner *
2066 learner_find(struct rte_swx_pipeline *p, const char *name);
2067
2068 static int
2069 instr_table_translate(struct rte_swx_pipeline *p,
2070                       struct action *action,
2071                       char **tokens,
2072                       int n_tokens,
2073                       struct instruction *instr,
2074                       struct instruction_data *data __rte_unused)
2075 {
2076         struct table *t;
2077         struct selector *s;
2078         struct learner *l;
2079
2080         CHECK(!action, EINVAL);
2081         CHECK(n_tokens == 2, EINVAL);
2082
2083         t = table_find(p, tokens[1]);
2084         if (t) {
2085                 instr->type = INSTR_TABLE;
2086                 instr->table.table_id = t->id;
2087                 return 0;
2088         }
2089
2090         s = selector_find(p, tokens[1]);
2091         if (s) {
2092                 instr->type = INSTR_SELECTOR;
2093                 instr->table.table_id = s->id;
2094                 return 0;
2095         }
2096
2097         l = learner_find(p, tokens[1]);
2098         if (l) {
2099                 instr->type = INSTR_LEARNER;
2100                 instr->table.table_id = l->id;
2101                 return 0;
2102         }
2103
2104         CHECK(0, EINVAL);
2105 }
2106
2107 static inline void
2108 instr_table_exec(struct rte_swx_pipeline *p)
2109 {
2110         struct thread *t = &p->threads[p->thread_id];
2111         struct instruction *ip = t->ip;
2112         uint32_t table_id = ip->table.table_id;
2113         struct rte_swx_table_state *ts = &t->table_state[table_id];
2114         struct table_runtime *table = &t->tables[table_id];
2115         struct table_statistics *stats = &p->table_stats[table_id];
2116         uint64_t action_id, n_pkts_hit, n_pkts_action;
2117         uint8_t *action_data;
2118         int done, hit;
2119
2120         /* Table. */
2121         done = table->func(ts->obj,
2122                            table->mailbox,
2123                            table->key,
2124                            &action_id,
2125                            &action_data,
2126                            &hit);
2127         if (!done) {
2128                 /* Thread. */
2129                 TRACE("[Thread %2u] table %u (not finalized)\n",
2130                       p->thread_id,
2131                       table_id);
2132
2133                 thread_yield(p);
2134                 return;
2135         }
2136
2137         action_id = hit ? action_id : ts->default_action_id;
2138         action_data = hit ? action_data : ts->default_action_data;
2139         n_pkts_hit = stats->n_pkts_hit[hit];
2140         n_pkts_action = stats->n_pkts_action[action_id];
2141
2142         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2143               p->thread_id,
2144               table_id,
2145               hit ? "hit" : "miss",
2146               (uint32_t)action_id);
2147
2148         t->action_id = action_id;
2149         t->structs[0] = action_data;
2150         t->hit = hit;
2151         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2152         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2153
2154         /* Thread. */
2155         thread_ip_action_call(p, t, action_id);
2156 }
2157
2158 static inline void
2159 instr_table_af_exec(struct rte_swx_pipeline *p)
2160 {
2161         struct thread *t = &p->threads[p->thread_id];
2162         struct instruction *ip = t->ip;
2163         uint32_t table_id = ip->table.table_id;
2164         struct rte_swx_table_state *ts = &t->table_state[table_id];
2165         struct table_runtime *table = &t->tables[table_id];
2166         struct table_statistics *stats = &p->table_stats[table_id];
2167         uint64_t action_id, n_pkts_hit, n_pkts_action;
2168         uint8_t *action_data;
2169         action_func_t action_func;
2170         int done, hit;
2171
2172         /* Table. */
2173         done = table->func(ts->obj,
2174                            table->mailbox,
2175                            table->key,
2176                            &action_id,
2177                            &action_data,
2178                            &hit);
2179         if (!done) {
2180                 /* Thread. */
2181                 TRACE("[Thread %2u] table %u (not finalized)\n",
2182                       p->thread_id,
2183                       table_id);
2184
2185                 thread_yield(p);
2186                 return;
2187         }
2188
2189         action_id = hit ? action_id : ts->default_action_id;
2190         action_data = hit ? action_data : ts->default_action_data;
2191         action_func = p->action_funcs[action_id];
2192         n_pkts_hit = stats->n_pkts_hit[hit];
2193         n_pkts_action = stats->n_pkts_action[action_id];
2194
2195         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2196               p->thread_id,
2197               table_id,
2198               hit ? "hit" : "miss",
2199               (uint32_t)action_id);
2200
2201         t->action_id = action_id;
2202         t->structs[0] = action_data;
2203         t->hit = hit;
2204         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2205         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2206
2207         /* Thread. */
2208         thread_ip_inc(p);
2209
2210         /* Action. */
2211         action_func(p);
2212 }
2213
2214 static inline void
2215 instr_selector_exec(struct rte_swx_pipeline *p)
2216 {
2217         struct thread *t = &p->threads[p->thread_id];
2218         struct instruction *ip = t->ip;
2219         uint32_t selector_id = ip->table.table_id;
2220         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2221         struct selector_runtime *selector = &t->selectors[selector_id];
2222         struct selector_statistics *stats = &p->selector_stats[selector_id];
2223         uint64_t n_pkts = stats->n_pkts;
2224         int done;
2225
2226         /* Table. */
2227         done = rte_swx_table_selector_select(ts->obj,
2228                            selector->mailbox,
2229                            selector->group_id_buffer,
2230                            selector->selector_buffer,
2231                            selector->member_id_buffer);
2232         if (!done) {
2233                 /* Thread. */
2234                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2235                       p->thread_id,
2236                       selector_id);
2237
2238                 thread_yield(p);
2239                 return;
2240         }
2241
2242
2243         TRACE("[Thread %2u] selector %u\n",
2244               p->thread_id,
2245               selector_id);
2246
2247         stats->n_pkts = n_pkts + 1;
2248
2249         /* Thread. */
2250         thread_ip_inc(p);
2251 }
2252
2253 static inline void
2254 instr_learner_exec(struct rte_swx_pipeline *p)
2255 {
2256         struct thread *t = &p->threads[p->thread_id];
2257         struct instruction *ip = t->ip;
2258         uint32_t learner_id = ip->table.table_id;
2259         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2260                 p->n_selectors + learner_id];
2261         struct learner_runtime *l = &t->learners[learner_id];
2262         struct learner_statistics *stats = &p->learner_stats[learner_id];
2263         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2264         uint8_t *action_data;
2265         int done, hit;
2266
2267         /* Table. */
2268         time = rte_get_tsc_cycles();
2269
2270         done = rte_swx_table_learner_lookup(ts->obj,
2271                                             l->mailbox,
2272                                             time,
2273                                             l->key,
2274                                             &action_id,
2275                                             &action_data,
2276                                             &hit);
2277         if (!done) {
2278                 /* Thread. */
2279                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2280                       p->thread_id,
2281                       learner_id);
2282
2283                 thread_yield(p);
2284                 return;
2285         }
2286
2287         action_id = hit ? action_id : ts->default_action_id;
2288         action_data = hit ? action_data : ts->default_action_data;
2289         n_pkts_hit = stats->n_pkts_hit[hit];
2290         n_pkts_action = stats->n_pkts_action[action_id];
2291
2292         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2293               p->thread_id,
2294               learner_id,
2295               hit ? "hit" : "miss",
2296               (uint32_t)action_id);
2297
2298         t->action_id = action_id;
2299         t->structs[0] = action_data;
2300         t->hit = hit;
2301         t->learner_id = learner_id;
2302         t->time = time;
2303         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2304         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2305
2306         /* Thread. */
2307         thread_ip_action_call(p, t, action_id);
2308 }
2309
2310 static inline void
2311 instr_learner_af_exec(struct rte_swx_pipeline *p)
2312 {
2313         struct thread *t = &p->threads[p->thread_id];
2314         struct instruction *ip = t->ip;
2315         uint32_t learner_id = ip->table.table_id;
2316         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2317                 p->n_selectors + learner_id];
2318         struct learner_runtime *l = &t->learners[learner_id];
2319         struct learner_statistics *stats = &p->learner_stats[learner_id];
2320         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2321         uint8_t *action_data;
2322         action_func_t action_func;
2323         int done, hit;
2324
2325         /* Table. */
2326         time = rte_get_tsc_cycles();
2327
2328         done = rte_swx_table_learner_lookup(ts->obj,
2329                                             l->mailbox,
2330                                             time,
2331                                             l->key,
2332                                             &action_id,
2333                                             &action_data,
2334                                             &hit);
2335         if (!done) {
2336                 /* Thread. */
2337                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2338                       p->thread_id,
2339                       learner_id);
2340
2341                 thread_yield(p);
2342                 return;
2343         }
2344
2345         action_id = hit ? action_id : ts->default_action_id;
2346         action_data = hit ? action_data : ts->default_action_data;
2347         action_func = p->action_funcs[action_id];
2348         n_pkts_hit = stats->n_pkts_hit[hit];
2349         n_pkts_action = stats->n_pkts_action[action_id];
2350
2351         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2352               p->thread_id,
2353               learner_id,
2354               hit ? "hit" : "miss",
2355               (uint32_t)action_id);
2356
2357         t->action_id = action_id;
2358         t->structs[0] = action_data;
2359         t->hit = hit;
2360         t->learner_id = learner_id;
2361         t->time = time;
2362         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2363         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2364
2365         /* Thread. */
2366         thread_ip_action_call(p, t, action_id);
2367
2368         /* Action */
2369         action_func(p);
2370 }
2371
2372 /*
2373  * learn.
2374  */
2375 static struct action *
2376 action_find(struct rte_swx_pipeline *p, const char *name);
2377
2378 static int
2379 action_has_nbo_args(struct action *a);
2380
2381 static int
2382 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2383
2384 static int
2385 instr_learn_translate(struct rte_swx_pipeline *p,
2386                       struct action *action,
2387                       char **tokens,
2388                       int n_tokens,
2389                       struct instruction *instr,
2390                       struct instruction_data *data __rte_unused)
2391 {
2392         struct action *a;
2393         const char *mf_name;
2394         uint32_t mf_offset = 0;
2395
2396         CHECK(action, EINVAL);
2397         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2398
2399         a = action_find(p, tokens[1]);
2400         CHECK(a, EINVAL);
2401         CHECK(!action_has_nbo_args(a), EINVAL);
2402
2403         mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2404         CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2405
2406         if (mf_name) {
2407                 struct field *mf;
2408
2409                 mf = metadata_field_parse(p, mf_name);
2410                 CHECK(mf, EINVAL);
2411
2412                 mf_offset = mf->offset / 8;
2413         }
2414
2415         instr->type = INSTR_LEARNER_LEARN;
2416         instr->learn.action_id = a->id;
2417         instr->learn.mf_offset = mf_offset;
2418
2419         return 0;
2420 }
2421
2422 static inline void
2423 instr_learn_exec(struct rte_swx_pipeline *p)
2424 {
2425         struct thread *t = &p->threads[p->thread_id];
2426         struct instruction *ip = t->ip;
2427
2428         __instr_learn_exec(p, t, ip);
2429
2430         /* Thread. */
2431         thread_ip_inc(p);
2432 }
2433
2434 /*
2435  * forget.
2436  */
2437 static int
2438 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2439                        struct action *action,
2440                        char **tokens __rte_unused,
2441                        int n_tokens,
2442                        struct instruction *instr,
2443                        struct instruction_data *data __rte_unused)
2444 {
2445         CHECK(action, EINVAL);
2446         CHECK(n_tokens == 1, EINVAL);
2447
2448         instr->type = INSTR_LEARNER_FORGET;
2449
2450         return 0;
2451 }
2452
2453 static inline void
2454 instr_forget_exec(struct rte_swx_pipeline *p)
2455 {
2456         struct thread *t = &p->threads[p->thread_id];
2457         struct instruction *ip = t->ip;
2458
2459         __instr_forget_exec(p, t, ip);
2460
2461         /* Thread. */
2462         thread_ip_inc(p);
2463 }
2464
2465 /*
2466  * extern.
2467  */
2468 static int
2469 instr_extern_translate(struct rte_swx_pipeline *p,
2470                        struct action *action __rte_unused,
2471                        char **tokens,
2472                        int n_tokens,
2473                        struct instruction *instr,
2474                        struct instruction_data *data __rte_unused)
2475 {
2476         char *token = tokens[1];
2477
2478         CHECK(n_tokens == 2, EINVAL);
2479
2480         if (token[0] == 'e') {
2481                 struct extern_obj *obj;
2482                 struct extern_type_member_func *func;
2483
2484                 func = extern_obj_member_func_parse(p, token, &obj);
2485                 CHECK(func, EINVAL);
2486
2487                 instr->type = INSTR_EXTERN_OBJ;
2488                 instr->ext_obj.ext_obj_id = obj->id;
2489                 instr->ext_obj.func_id = func->id;
2490
2491                 return 0;
2492         }
2493
2494         if (token[0] == 'f') {
2495                 struct extern_func *func;
2496
2497                 func = extern_func_parse(p, token);
2498                 CHECK(func, EINVAL);
2499
2500                 instr->type = INSTR_EXTERN_FUNC;
2501                 instr->ext_func.ext_func_id = func->id;
2502
2503                 return 0;
2504         }
2505
2506         CHECK(0, EINVAL);
2507 }
2508
2509 static inline void
2510 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2511 {
2512         struct thread *t = &p->threads[p->thread_id];
2513         struct instruction *ip = t->ip;
2514         uint32_t done;
2515
2516         /* Extern object member function execute. */
2517         done = __instr_extern_obj_exec(p, t, ip);
2518
2519         /* Thread. */
2520         thread_ip_inc_cond(t, done);
2521         thread_yield_cond(p, done ^ 1);
2522 }
2523
2524 static inline void
2525 instr_extern_func_exec(struct rte_swx_pipeline *p)
2526 {
2527         struct thread *t = &p->threads[p->thread_id];
2528         struct instruction *ip = t->ip;
2529         uint32_t done;
2530
2531         /* Extern function execute. */
2532         done = __instr_extern_func_exec(p, t, ip);
2533
2534         /* Thread. */
2535         thread_ip_inc_cond(t, done);
2536         thread_yield_cond(p, done ^ 1);
2537 }
2538
2539 /*
2540  * mov.
2541  */
2542 static int
2543 instr_mov_translate(struct rte_swx_pipeline *p,
2544                     struct action *action,
2545                     char **tokens,
2546                     int n_tokens,
2547                     struct instruction *instr,
2548                     struct instruction_data *data __rte_unused)
2549 {
2550         char *dst = tokens[1], *src = tokens[2];
2551         struct field *fdst, *fsrc;
2552         uint64_t src_val;
2553         uint32_t dst_struct_id = 0, src_struct_id = 0;
2554
2555         CHECK(n_tokens == 3, EINVAL);
2556
2557         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2558         CHECK(fdst, EINVAL);
2559         CHECK(!fdst->var_size, EINVAL);
2560
2561         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2562         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2563         if (fsrc) {
2564                 CHECK(!fsrc->var_size, EINVAL);
2565
2566                 instr->type = INSTR_MOV;
2567                 if (dst[0] != 'h' && src[0] == 'h')
2568                         instr->type = INSTR_MOV_MH;
2569                 if (dst[0] == 'h' && src[0] != 'h')
2570                         instr->type = INSTR_MOV_HM;
2571                 if (dst[0] == 'h' && src[0] == 'h')
2572                         instr->type = INSTR_MOV_HH;
2573
2574                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2575                 instr->mov.dst.n_bits = fdst->n_bits;
2576                 instr->mov.dst.offset = fdst->offset / 8;
2577                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2578                 instr->mov.src.n_bits = fsrc->n_bits;
2579                 instr->mov.src.offset = fsrc->offset / 8;
2580                 return 0;
2581         }
2582
2583         /* MOV_I. */
2584         src_val = strtoull(src, &src, 0);
2585         CHECK(!src[0], EINVAL);
2586
2587         if (dst[0] == 'h')
2588                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2589
2590         instr->type = INSTR_MOV_I;
2591         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2592         instr->mov.dst.n_bits = fdst->n_bits;
2593         instr->mov.dst.offset = fdst->offset / 8;
2594         instr->mov.src_val = src_val;
2595         return 0;
2596 }
2597
2598 static inline void
2599 instr_mov_exec(struct rte_swx_pipeline *p)
2600 {
2601         struct thread *t = &p->threads[p->thread_id];
2602         struct instruction *ip = t->ip;
2603
2604         __instr_mov_exec(p, t, ip);
2605
2606         /* Thread. */
2607         thread_ip_inc(p);
2608 }
2609
2610 static inline void
2611 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2612 {
2613         struct thread *t = &p->threads[p->thread_id];
2614         struct instruction *ip = t->ip;
2615
2616         __instr_mov_mh_exec(p, t, ip);
2617
2618         /* Thread. */
2619         thread_ip_inc(p);
2620 }
2621
2622 static inline void
2623 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2624 {
2625         struct thread *t = &p->threads[p->thread_id];
2626         struct instruction *ip = t->ip;
2627
2628         __instr_mov_hm_exec(p, t, ip);
2629
2630         /* Thread. */
2631         thread_ip_inc(p);
2632 }
2633
2634 static inline void
2635 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2636 {
2637         struct thread *t = &p->threads[p->thread_id];
2638         struct instruction *ip = t->ip;
2639
2640         __instr_mov_hh_exec(p, t, ip);
2641
2642         /* Thread. */
2643         thread_ip_inc(p);
2644 }
2645
2646 static inline void
2647 instr_mov_i_exec(struct rte_swx_pipeline *p)
2648 {
2649         struct thread *t = &p->threads[p->thread_id];
2650         struct instruction *ip = t->ip;
2651
2652         __instr_mov_i_exec(p, t, ip);
2653
2654         /* Thread. */
2655         thread_ip_inc(p);
2656 }
2657
2658 /*
2659  * dma.
2660  */
2661 static inline void
2662 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2663 {
2664         struct thread *t = &p->threads[p->thread_id];
2665         struct instruction *ip = t->ip;
2666
2667         __instr_dma_ht_exec(p, t, ip);
2668
2669         /* Thread. */
2670         thread_ip_inc(p);
2671 }
2672
2673 static inline void
2674 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2675 {
2676         struct thread *t = &p->threads[p->thread_id];
2677         struct instruction *ip = t->ip;
2678
2679         __instr_dma_ht2_exec(p, t, ip);
2680
2681         /* Thread. */
2682         thread_ip_inc(p);
2683 }
2684
2685 static inline void
2686 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2687 {
2688         struct thread *t = &p->threads[p->thread_id];
2689         struct instruction *ip = t->ip;
2690
2691         __instr_dma_ht3_exec(p, t, ip);
2692
2693         /* Thread. */
2694         thread_ip_inc(p);
2695 }
2696
2697 static inline void
2698 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2699 {
2700         struct thread *t = &p->threads[p->thread_id];
2701         struct instruction *ip = t->ip;
2702
2703         __instr_dma_ht4_exec(p, t, ip);
2704
2705         /* Thread. */
2706         thread_ip_inc(p);
2707 }
2708
2709 static inline void
2710 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2711 {
2712         struct thread *t = &p->threads[p->thread_id];
2713         struct instruction *ip = t->ip;
2714
2715         __instr_dma_ht5_exec(p, t, ip);
2716
2717         /* Thread. */
2718         thread_ip_inc(p);
2719 }
2720
2721 static inline void
2722 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2723 {
2724         struct thread *t = &p->threads[p->thread_id];
2725         struct instruction *ip = t->ip;
2726
2727         __instr_dma_ht6_exec(p, t, ip);
2728
2729         /* Thread. */
2730         thread_ip_inc(p);
2731 }
2732
2733 static inline void
2734 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2735 {
2736         struct thread *t = &p->threads[p->thread_id];
2737         struct instruction *ip = t->ip;
2738
2739         __instr_dma_ht7_exec(p, t, ip);
2740
2741         /* Thread. */
2742         thread_ip_inc(p);
2743 }
2744
2745 static inline void
2746 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2747 {
2748         struct thread *t = &p->threads[p->thread_id];
2749         struct instruction *ip = t->ip;
2750
2751         __instr_dma_ht8_exec(p, t, ip);
2752
2753         /* Thread. */
2754         thread_ip_inc(p);
2755 }
2756
2757 /*
2758  * alu.
2759  */
2760 static int
2761 instr_alu_add_translate(struct rte_swx_pipeline *p,
2762                         struct action *action,
2763                         char **tokens,
2764                         int n_tokens,
2765                         struct instruction *instr,
2766                         struct instruction_data *data __rte_unused)
2767 {
2768         char *dst = tokens[1], *src = tokens[2];
2769         struct field *fdst, *fsrc;
2770         uint64_t src_val;
2771         uint32_t dst_struct_id = 0, src_struct_id = 0;
2772
2773         CHECK(n_tokens == 3, EINVAL);
2774
2775         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2776         CHECK(fdst, EINVAL);
2777         CHECK(!fdst->var_size, EINVAL);
2778
2779         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2780         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2781         if (fsrc) {
2782                 CHECK(!fsrc->var_size, EINVAL);
2783
2784                 instr->type = INSTR_ALU_ADD;
2785                 if (dst[0] == 'h' && src[0] != 'h')
2786                         instr->type = INSTR_ALU_ADD_HM;
2787                 if (dst[0] != 'h' && src[0] == 'h')
2788                         instr->type = INSTR_ALU_ADD_MH;
2789                 if (dst[0] == 'h' && src[0] == 'h')
2790                         instr->type = INSTR_ALU_ADD_HH;
2791
2792                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2793                 instr->alu.dst.n_bits = fdst->n_bits;
2794                 instr->alu.dst.offset = fdst->offset / 8;
2795                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2796                 instr->alu.src.n_bits = fsrc->n_bits;
2797                 instr->alu.src.offset = fsrc->offset / 8;
2798                 return 0;
2799         }
2800
2801         /* ADD_MI, ADD_HI. */
2802         src_val = strtoull(src, &src, 0);
2803         CHECK(!src[0], EINVAL);
2804
2805         instr->type = INSTR_ALU_ADD_MI;
2806         if (dst[0] == 'h')
2807                 instr->type = INSTR_ALU_ADD_HI;
2808
2809         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2810         instr->alu.dst.n_bits = fdst->n_bits;
2811         instr->alu.dst.offset = fdst->offset / 8;
2812         instr->alu.src_val = src_val;
2813         return 0;
2814 }
2815
2816 static int
2817 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2818                         struct action *action,
2819                         char **tokens,
2820                         int n_tokens,
2821                         struct instruction *instr,
2822                         struct instruction_data *data __rte_unused)
2823 {
2824         char *dst = tokens[1], *src = tokens[2];
2825         struct field *fdst, *fsrc;
2826         uint64_t src_val;
2827         uint32_t dst_struct_id = 0, src_struct_id = 0;
2828
2829         CHECK(n_tokens == 3, EINVAL);
2830
2831         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2832         CHECK(fdst, EINVAL);
2833         CHECK(!fdst->var_size, EINVAL);
2834
2835         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2836         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2837         if (fsrc) {
2838                 CHECK(!fsrc->var_size, EINVAL);
2839
2840                 instr->type = INSTR_ALU_SUB;
2841                 if (dst[0] == 'h' && src[0] != 'h')
2842                         instr->type = INSTR_ALU_SUB_HM;
2843                 if (dst[0] != 'h' && src[0] == 'h')
2844                         instr->type = INSTR_ALU_SUB_MH;
2845                 if (dst[0] == 'h' && src[0] == 'h')
2846                         instr->type = INSTR_ALU_SUB_HH;
2847
2848                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2849                 instr->alu.dst.n_bits = fdst->n_bits;
2850                 instr->alu.dst.offset = fdst->offset / 8;
2851                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2852                 instr->alu.src.n_bits = fsrc->n_bits;
2853                 instr->alu.src.offset = fsrc->offset / 8;
2854                 return 0;
2855         }
2856
2857         /* SUB_MI, SUB_HI. */
2858         src_val = strtoull(src, &src, 0);
2859         CHECK(!src[0], EINVAL);
2860
2861         instr->type = INSTR_ALU_SUB_MI;
2862         if (dst[0] == 'h')
2863                 instr->type = INSTR_ALU_SUB_HI;
2864
2865         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2866         instr->alu.dst.n_bits = fdst->n_bits;
2867         instr->alu.dst.offset = fdst->offset / 8;
2868         instr->alu.src_val = src_val;
2869         return 0;
2870 }
2871
2872 static int
2873 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2874                           struct action *action __rte_unused,
2875                           char **tokens,
2876                           int n_tokens,
2877                           struct instruction *instr,
2878                           struct instruction_data *data __rte_unused)
2879 {
2880         char *dst = tokens[1], *src = tokens[2];
2881         struct header *hdst, *hsrc;
2882         struct field *fdst, *fsrc;
2883
2884         CHECK(n_tokens == 3, EINVAL);
2885
2886         fdst = header_field_parse(p, dst, &hdst);
2887         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2888         CHECK(!fdst->var_size, EINVAL);
2889
2890         /* CKADD_FIELD. */
2891         fsrc = header_field_parse(p, src, &hsrc);
2892         if (fsrc) {
2893                 CHECK(!fsrc->var_size, EINVAL);
2894
2895                 instr->type = INSTR_ALU_CKADD_FIELD;
2896                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2897                 instr->alu.dst.n_bits = fdst->n_bits;
2898                 instr->alu.dst.offset = fdst->offset / 8;
2899                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2900                 instr->alu.src.n_bits = fsrc->n_bits;
2901                 instr->alu.src.offset = fsrc->offset / 8;
2902                 return 0;
2903         }
2904
2905         /* CKADD_STRUCT, CKADD_STRUCT20. */
2906         hsrc = header_parse(p, src);
2907         CHECK(hsrc, EINVAL);
2908         CHECK(!hsrc->st->var_size, EINVAL);
2909
2910         instr->type = INSTR_ALU_CKADD_STRUCT;
2911         if ((hsrc->st->n_bits / 8) == 20)
2912                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2913
2914         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2915         instr->alu.dst.n_bits = fdst->n_bits;
2916         instr->alu.dst.offset = fdst->offset / 8;
2917         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2918         instr->alu.src.n_bits = hsrc->st->n_bits;
2919         instr->alu.src.offset = 0; /* Unused. */
2920         return 0;
2921 }
2922
2923 static int
2924 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2925                           struct action *action __rte_unused,
2926                           char **tokens,
2927                           int n_tokens,
2928                           struct instruction *instr,
2929                           struct instruction_data *data __rte_unused)
2930 {
2931         char *dst = tokens[1], *src = tokens[2];
2932         struct header *hdst, *hsrc;
2933         struct field *fdst, *fsrc;
2934
2935         CHECK(n_tokens == 3, EINVAL);
2936
2937         fdst = header_field_parse(p, dst, &hdst);
2938         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2939         CHECK(!fdst->var_size, EINVAL);
2940
2941         fsrc = header_field_parse(p, src, &hsrc);
2942         CHECK(fsrc, EINVAL);
2943         CHECK(!fsrc->var_size, EINVAL);
2944
2945         instr->type = INSTR_ALU_CKSUB_FIELD;
2946         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2947         instr->alu.dst.n_bits = fdst->n_bits;
2948         instr->alu.dst.offset = fdst->offset / 8;
2949         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2950         instr->alu.src.n_bits = fsrc->n_bits;
2951         instr->alu.src.offset = fsrc->offset / 8;
2952         return 0;
2953 }
2954
2955 static int
2956 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2957                         struct action *action,
2958                         char **tokens,
2959                         int n_tokens,
2960                         struct instruction *instr,
2961                         struct instruction_data *data __rte_unused)
2962 {
2963         char *dst = tokens[1], *src = tokens[2];
2964         struct field *fdst, *fsrc;
2965         uint64_t src_val;
2966         uint32_t dst_struct_id = 0, src_struct_id = 0;
2967
2968         CHECK(n_tokens == 3, EINVAL);
2969
2970         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2971         CHECK(fdst, EINVAL);
2972         CHECK(!fdst->var_size, EINVAL);
2973
2974         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2975         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2976         if (fsrc) {
2977                 CHECK(!fsrc->var_size, EINVAL);
2978
2979                 instr->type = INSTR_ALU_SHL;
2980                 if (dst[0] == 'h' && src[0] != 'h')
2981                         instr->type = INSTR_ALU_SHL_HM;
2982                 if (dst[0] != 'h' && src[0] == 'h')
2983                         instr->type = INSTR_ALU_SHL_MH;
2984                 if (dst[0] == 'h' && src[0] == 'h')
2985                         instr->type = INSTR_ALU_SHL_HH;
2986
2987                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2988                 instr->alu.dst.n_bits = fdst->n_bits;
2989                 instr->alu.dst.offset = fdst->offset / 8;
2990                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2991                 instr->alu.src.n_bits = fsrc->n_bits;
2992                 instr->alu.src.offset = fsrc->offset / 8;
2993                 return 0;
2994         }
2995
2996         /* SHL_MI, SHL_HI. */
2997         src_val = strtoull(src, &src, 0);
2998         CHECK(!src[0], EINVAL);
2999
3000         instr->type = INSTR_ALU_SHL_MI;
3001         if (dst[0] == 'h')
3002                 instr->type = INSTR_ALU_SHL_HI;
3003
3004         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3005         instr->alu.dst.n_bits = fdst->n_bits;
3006         instr->alu.dst.offset = fdst->offset / 8;
3007         instr->alu.src_val = src_val;
3008         return 0;
3009 }
3010
3011 static int
3012 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3013                         struct action *action,
3014                         char **tokens,
3015                         int n_tokens,
3016                         struct instruction *instr,
3017                         struct instruction_data *data __rte_unused)
3018 {
3019         char *dst = tokens[1], *src = tokens[2];
3020         struct field *fdst, *fsrc;
3021         uint64_t src_val;
3022         uint32_t dst_struct_id = 0, src_struct_id = 0;
3023
3024         CHECK(n_tokens == 3, EINVAL);
3025
3026         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3027         CHECK(fdst, EINVAL);
3028         CHECK(!fdst->var_size, EINVAL);
3029
3030         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3031         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3032         if (fsrc) {
3033                 CHECK(!fsrc->var_size, EINVAL);
3034
3035                 instr->type = INSTR_ALU_SHR;
3036                 if (dst[0] == 'h' && src[0] != 'h')
3037                         instr->type = INSTR_ALU_SHR_HM;
3038                 if (dst[0] != 'h' && src[0] == 'h')
3039                         instr->type = INSTR_ALU_SHR_MH;
3040                 if (dst[0] == 'h' && src[0] == 'h')
3041                         instr->type = INSTR_ALU_SHR_HH;
3042
3043                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3044                 instr->alu.dst.n_bits = fdst->n_bits;
3045                 instr->alu.dst.offset = fdst->offset / 8;
3046                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3047                 instr->alu.src.n_bits = fsrc->n_bits;
3048                 instr->alu.src.offset = fsrc->offset / 8;
3049                 return 0;
3050         }
3051
3052         /* SHR_MI, SHR_HI. */
3053         src_val = strtoull(src, &src, 0);
3054         CHECK(!src[0], EINVAL);
3055
3056         instr->type = INSTR_ALU_SHR_MI;
3057         if (dst[0] == 'h')
3058                 instr->type = INSTR_ALU_SHR_HI;
3059
3060         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3061         instr->alu.dst.n_bits = fdst->n_bits;
3062         instr->alu.dst.offset = fdst->offset / 8;
3063         instr->alu.src_val = src_val;
3064         return 0;
3065 }
3066
3067 static int
3068 instr_alu_and_translate(struct rte_swx_pipeline *p,
3069                         struct action *action,
3070                         char **tokens,
3071                         int n_tokens,
3072                         struct instruction *instr,
3073                         struct instruction_data *data __rte_unused)
3074 {
3075         char *dst = tokens[1], *src = tokens[2];
3076         struct field *fdst, *fsrc;
3077         uint64_t src_val;
3078         uint32_t dst_struct_id = 0, src_struct_id = 0;
3079
3080         CHECK(n_tokens == 3, EINVAL);
3081
3082         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3083         CHECK(fdst, EINVAL);
3084         CHECK(!fdst->var_size, EINVAL);
3085
3086         /* AND, AND_MH, AND_HM, AND_HH. */
3087         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3088         if (fsrc) {
3089                 CHECK(!fsrc->var_size, EINVAL);
3090
3091                 instr->type = INSTR_ALU_AND;
3092                 if (dst[0] != 'h' && src[0] == 'h')
3093                         instr->type = INSTR_ALU_AND_MH;
3094                 if (dst[0] == 'h' && src[0] != 'h')
3095                         instr->type = INSTR_ALU_AND_HM;
3096                 if (dst[0] == 'h' && src[0] == 'h')
3097                         instr->type = INSTR_ALU_AND_HH;
3098
3099                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3100                 instr->alu.dst.n_bits = fdst->n_bits;
3101                 instr->alu.dst.offset = fdst->offset / 8;
3102                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3103                 instr->alu.src.n_bits = fsrc->n_bits;
3104                 instr->alu.src.offset = fsrc->offset / 8;
3105                 return 0;
3106         }
3107
3108         /* AND_I. */
3109         src_val = strtoull(src, &src, 0);
3110         CHECK(!src[0], EINVAL);
3111
3112         if (dst[0] == 'h')
3113                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3114
3115         instr->type = INSTR_ALU_AND_I;
3116         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3117         instr->alu.dst.n_bits = fdst->n_bits;
3118         instr->alu.dst.offset = fdst->offset / 8;
3119         instr->alu.src_val = src_val;
3120         return 0;
3121 }
3122
3123 static int
3124 instr_alu_or_translate(struct rte_swx_pipeline *p,
3125                        struct action *action,
3126                        char **tokens,
3127                        int n_tokens,
3128                        struct instruction *instr,
3129                        struct instruction_data *data __rte_unused)
3130 {
3131         char *dst = tokens[1], *src = tokens[2];
3132         struct field *fdst, *fsrc;
3133         uint64_t src_val;
3134         uint32_t dst_struct_id = 0, src_struct_id = 0;
3135
3136         CHECK(n_tokens == 3, EINVAL);
3137
3138         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3139         CHECK(fdst, EINVAL);
3140         CHECK(!fdst->var_size, EINVAL);
3141
3142         /* OR, OR_MH, OR_HM, OR_HH. */
3143         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3144         if (fsrc) {
3145                 CHECK(!fsrc->var_size, EINVAL);
3146
3147                 instr->type = INSTR_ALU_OR;
3148                 if (dst[0] != 'h' && src[0] == 'h')
3149                         instr->type = INSTR_ALU_OR_MH;
3150                 if (dst[0] == 'h' && src[0] != 'h')
3151                         instr->type = INSTR_ALU_OR_HM;
3152                 if (dst[0] == 'h' && src[0] == 'h')
3153                         instr->type = INSTR_ALU_OR_HH;
3154
3155                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3156                 instr->alu.dst.n_bits = fdst->n_bits;
3157                 instr->alu.dst.offset = fdst->offset / 8;
3158                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3159                 instr->alu.src.n_bits = fsrc->n_bits;
3160                 instr->alu.src.offset = fsrc->offset / 8;
3161                 return 0;
3162         }
3163
3164         /* OR_I. */
3165         src_val = strtoull(src, &src, 0);
3166         CHECK(!src[0], EINVAL);
3167
3168         if (dst[0] == 'h')
3169                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3170
3171         instr->type = INSTR_ALU_OR_I;
3172         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3173         instr->alu.dst.n_bits = fdst->n_bits;
3174         instr->alu.dst.offset = fdst->offset / 8;
3175         instr->alu.src_val = src_val;
3176         return 0;
3177 }
3178
3179 static int
3180 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3181                         struct action *action,
3182                         char **tokens,
3183                         int n_tokens,
3184                         struct instruction *instr,
3185                         struct instruction_data *data __rte_unused)
3186 {
3187         char *dst = tokens[1], *src = tokens[2];
3188         struct field *fdst, *fsrc;
3189         uint64_t src_val;
3190         uint32_t dst_struct_id = 0, src_struct_id = 0;
3191
3192         CHECK(n_tokens == 3, EINVAL);
3193
3194         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3195         CHECK(fdst, EINVAL);
3196         CHECK(!fdst->var_size, EINVAL);
3197
3198         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3199         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3200         if (fsrc) {
3201                 CHECK(!fsrc->var_size, EINVAL);
3202
3203                 instr->type = INSTR_ALU_XOR;
3204                 if (dst[0] != 'h' && src[0] == 'h')
3205                         instr->type = INSTR_ALU_XOR_MH;
3206                 if (dst[0] == 'h' && src[0] != 'h')
3207                         instr->type = INSTR_ALU_XOR_HM;
3208                 if (dst[0] == 'h' && src[0] == 'h')
3209                         instr->type = INSTR_ALU_XOR_HH;
3210
3211                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3212                 instr->alu.dst.n_bits = fdst->n_bits;
3213                 instr->alu.dst.offset = fdst->offset / 8;
3214                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3215                 instr->alu.src.n_bits = fsrc->n_bits;
3216                 instr->alu.src.offset = fsrc->offset / 8;
3217                 return 0;
3218         }
3219
3220         /* XOR_I. */
3221         src_val = strtoull(src, &src, 0);
3222         CHECK(!src[0], EINVAL);
3223
3224         if (dst[0] == 'h')
3225                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3226
3227         instr->type = INSTR_ALU_XOR_I;
3228         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3229         instr->alu.dst.n_bits = fdst->n_bits;
3230         instr->alu.dst.offset = fdst->offset / 8;
3231         instr->alu.src_val = src_val;
3232         return 0;
3233 }
3234
3235 static inline void
3236 instr_alu_add_exec(struct rte_swx_pipeline *p)
3237 {
3238         struct thread *t = &p->threads[p->thread_id];
3239         struct instruction *ip = t->ip;
3240
3241         /* Structs */
3242         __instr_alu_add_exec(p, t, ip);
3243
3244         /* Thread. */
3245         thread_ip_inc(p);
3246 }
3247
3248 static inline void
3249 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3250 {
3251         struct thread *t = &p->threads[p->thread_id];
3252         struct instruction *ip = t->ip;
3253
3254         /* Structs. */
3255         __instr_alu_add_mh_exec(p, t, ip);
3256
3257         /* Thread. */
3258         thread_ip_inc(p);
3259 }
3260
3261 static inline void
3262 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3263 {
3264         struct thread *t = &p->threads[p->thread_id];
3265         struct instruction *ip = t->ip;
3266
3267         /* Structs. */
3268         __instr_alu_add_hm_exec(p, t, ip);
3269
3270         /* Thread. */
3271         thread_ip_inc(p);
3272 }
3273
3274 static inline void
3275 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3276 {
3277         struct thread *t = &p->threads[p->thread_id];
3278         struct instruction *ip = t->ip;
3279
3280         /* Structs. */
3281         __instr_alu_add_hh_exec(p, t, ip);
3282
3283         /* Thread. */
3284         thread_ip_inc(p);
3285 }
3286
3287 static inline void
3288 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3289 {
3290         struct thread *t = &p->threads[p->thread_id];
3291         struct instruction *ip = t->ip;
3292
3293         /* Structs. */
3294         __instr_alu_add_mi_exec(p, t, ip);
3295
3296         /* Thread. */
3297         thread_ip_inc(p);
3298 }
3299
3300 static inline void
3301 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3302 {
3303         struct thread *t = &p->threads[p->thread_id];
3304         struct instruction *ip = t->ip;
3305
3306         /* Structs. */
3307         __instr_alu_add_hi_exec(p, t, ip);
3308
3309         /* Thread. */
3310         thread_ip_inc(p);
3311 }
3312
3313 static inline void
3314 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3315 {
3316         struct thread *t = &p->threads[p->thread_id];
3317         struct instruction *ip = t->ip;
3318
3319         /* Structs. */
3320         __instr_alu_sub_exec(p, t, ip);
3321
3322         /* Thread. */
3323         thread_ip_inc(p);
3324 }
3325
3326 static inline void
3327 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3328 {
3329         struct thread *t = &p->threads[p->thread_id];
3330         struct instruction *ip = t->ip;
3331
3332         /* Structs. */
3333         __instr_alu_sub_mh_exec(p, t, ip);
3334
3335         /* Thread. */
3336         thread_ip_inc(p);
3337 }
3338
3339 static inline void
3340 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3341 {
3342         struct thread *t = &p->threads[p->thread_id];
3343         struct instruction *ip = t->ip;
3344
3345         /* Structs. */
3346         __instr_alu_sub_hm_exec(p, t, ip);
3347
3348         /* Thread. */
3349         thread_ip_inc(p);
3350 }
3351
3352 static inline void
3353 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3354 {
3355         struct thread *t = &p->threads[p->thread_id];
3356         struct instruction *ip = t->ip;
3357
3358         /* Structs. */
3359         __instr_alu_sub_hh_exec(p, t, ip);
3360
3361         /* Thread. */
3362         thread_ip_inc(p);
3363 }
3364
3365 static inline void
3366 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3367 {
3368         struct thread *t = &p->threads[p->thread_id];
3369         struct instruction *ip = t->ip;
3370
3371         /* Structs. */
3372         __instr_alu_sub_mi_exec(p, t, ip);
3373
3374         /* Thread. */
3375         thread_ip_inc(p);
3376 }
3377
3378 static inline void
3379 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3380 {
3381         struct thread *t = &p->threads[p->thread_id];
3382         struct instruction *ip = t->ip;
3383
3384         /* Structs. */
3385         __instr_alu_sub_hi_exec(p, t, ip);
3386
3387         /* Thread. */
3388         thread_ip_inc(p);
3389 }
3390
3391 static inline void
3392 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3393 {
3394         struct thread *t = &p->threads[p->thread_id];
3395         struct instruction *ip = t->ip;
3396
3397         /* Structs. */
3398         __instr_alu_shl_exec(p, t, ip);
3399
3400         /* Thread. */
3401         thread_ip_inc(p);
3402 }
3403
3404 static inline void
3405 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3406 {
3407         struct thread *t = &p->threads[p->thread_id];
3408         struct instruction *ip = t->ip;
3409
3410         /* Structs. */
3411         __instr_alu_shl_mh_exec(p, t, ip);
3412
3413         /* Thread. */
3414         thread_ip_inc(p);
3415 }
3416
3417 static inline void
3418 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3419 {
3420         struct thread *t = &p->threads[p->thread_id];
3421         struct instruction *ip = t->ip;
3422
3423         /* Structs. */
3424         __instr_alu_shl_hm_exec(p, t, ip);
3425
3426         /* Thread. */
3427         thread_ip_inc(p);
3428 }
3429
3430 static inline void
3431 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3432 {
3433         struct thread *t = &p->threads[p->thread_id];
3434         struct instruction *ip = t->ip;
3435
3436         /* Structs. */
3437         __instr_alu_shl_hh_exec(p, t, ip);
3438
3439         /* Thread. */
3440         thread_ip_inc(p);
3441 }
3442
3443 static inline void
3444 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3445 {
3446         struct thread *t = &p->threads[p->thread_id];
3447         struct instruction *ip = t->ip;
3448
3449         /* Structs. */
3450         __instr_alu_shl_mi_exec(p, t, ip);
3451
3452         /* Thread. */
3453         thread_ip_inc(p);
3454 }
3455
3456 static inline void
3457 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3458 {
3459         struct thread *t = &p->threads[p->thread_id];
3460         struct instruction *ip = t->ip;
3461
3462         /* Structs. */
3463         __instr_alu_shl_hi_exec(p, t, ip);
3464
3465         /* Thread. */
3466         thread_ip_inc(p);
3467 }
3468
3469 static inline void
3470 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3471 {
3472         struct thread *t = &p->threads[p->thread_id];
3473         struct instruction *ip = t->ip;
3474
3475         /* Structs. */
3476         __instr_alu_shr_exec(p, t, ip);
3477
3478         /* Thread. */
3479         thread_ip_inc(p);
3480 }
3481
3482 static inline void
3483 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3484 {
3485         struct thread *t = &p->threads[p->thread_id];
3486         struct instruction *ip = t->ip;
3487
3488         /* Structs. */
3489         __instr_alu_shr_mh_exec(p, t, ip);
3490
3491         /* Thread. */
3492         thread_ip_inc(p);
3493 }
3494
3495 static inline void
3496 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3497 {
3498         struct thread *t = &p->threads[p->thread_id];
3499         struct instruction *ip = t->ip;
3500
3501         /* Structs. */
3502         __instr_alu_shr_hm_exec(p, t, ip);
3503
3504         /* Thread. */
3505         thread_ip_inc(p);
3506 }
3507
3508 static inline void
3509 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3510 {
3511         struct thread *t = &p->threads[p->thread_id];
3512         struct instruction *ip = t->ip;
3513
3514         /* Structs. */
3515         __instr_alu_shr_hh_exec(p, t, ip);
3516
3517         /* Thread. */
3518         thread_ip_inc(p);
3519 }
3520
3521 static inline void
3522 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3523 {
3524         struct thread *t = &p->threads[p->thread_id];
3525         struct instruction *ip = t->ip;
3526
3527         /* Structs. */
3528         __instr_alu_shr_mi_exec(p, t, ip);
3529
3530         /* Thread. */
3531         thread_ip_inc(p);
3532 }
3533
3534 static inline void
3535 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3536 {
3537         struct thread *t = &p->threads[p->thread_id];
3538         struct instruction *ip = t->ip;
3539
3540         /* Structs. */
3541         __instr_alu_shr_hi_exec(p, t, ip);
3542
3543         /* Thread. */
3544         thread_ip_inc(p);
3545 }
3546
3547 static inline void
3548 instr_alu_and_exec(struct rte_swx_pipeline *p)
3549 {
3550         struct thread *t = &p->threads[p->thread_id];
3551         struct instruction *ip = t->ip;
3552
3553         /* Structs. */
3554         __instr_alu_and_exec(p, t, ip);
3555
3556         /* Thread. */
3557         thread_ip_inc(p);
3558 }
3559
3560 static inline void
3561 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3562 {
3563         struct thread *t = &p->threads[p->thread_id];
3564         struct instruction *ip = t->ip;
3565
3566         /* Structs. */
3567         __instr_alu_and_mh_exec(p, t, ip);
3568
3569         /* Thread. */
3570         thread_ip_inc(p);
3571 }
3572
3573 static inline void
3574 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3575 {
3576         struct thread *t = &p->threads[p->thread_id];
3577         struct instruction *ip = t->ip;
3578
3579         /* Structs. */
3580         __instr_alu_and_hm_exec(p, t, ip);
3581
3582         /* Thread. */
3583         thread_ip_inc(p);
3584 }
3585
3586 static inline void
3587 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3588 {
3589         struct thread *t = &p->threads[p->thread_id];
3590         struct instruction *ip = t->ip;
3591
3592         /* Structs. */
3593         __instr_alu_and_hh_exec(p, t, ip);
3594
3595         /* Thread. */
3596         thread_ip_inc(p);
3597 }
3598
3599 static inline void
3600 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3601 {
3602         struct thread *t = &p->threads[p->thread_id];
3603         struct instruction *ip = t->ip;
3604
3605         /* Structs. */
3606         __instr_alu_and_i_exec(p, t, ip);
3607
3608         /* Thread. */
3609         thread_ip_inc(p);
3610 }
3611
3612 static inline void
3613 instr_alu_or_exec(struct rte_swx_pipeline *p)
3614 {
3615         struct thread *t = &p->threads[p->thread_id];
3616         struct instruction *ip = t->ip;
3617
3618         /* Structs. */
3619         __instr_alu_or_exec(p, t, ip);
3620
3621         /* Thread. */
3622         thread_ip_inc(p);
3623 }
3624
3625 static inline void
3626 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3627 {
3628         struct thread *t = &p->threads[p->thread_id];
3629         struct instruction *ip = t->ip;
3630
3631         /* Structs. */
3632         __instr_alu_or_mh_exec(p, t, ip);
3633
3634         /* Thread. */
3635         thread_ip_inc(p);
3636 }
3637
3638 static inline void
3639 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3640 {
3641         struct thread *t = &p->threads[p->thread_id];
3642         struct instruction *ip = t->ip;
3643
3644         /* Structs. */
3645         __instr_alu_or_hm_exec(p, t, ip);
3646
3647         /* Thread. */
3648         thread_ip_inc(p);
3649 }
3650
3651 static inline void
3652 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3653 {
3654         struct thread *t = &p->threads[p->thread_id];
3655         struct instruction *ip = t->ip;
3656
3657         /* Structs. */
3658         __instr_alu_or_hh_exec(p, t, ip);
3659
3660         /* Thread. */
3661         thread_ip_inc(p);
3662 }
3663
3664 static inline void
3665 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3666 {
3667         struct thread *t = &p->threads[p->thread_id];
3668         struct instruction *ip = t->ip;
3669
3670         /* Structs. */
3671         __instr_alu_or_i_exec(p, t, ip);
3672
3673         /* Thread. */
3674         thread_ip_inc(p);
3675 }
3676
3677 static inline void
3678 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3679 {
3680         struct thread *t = &p->threads[p->thread_id];
3681         struct instruction *ip = t->ip;
3682
3683         /* Structs. */
3684         __instr_alu_xor_exec(p, t, ip);
3685
3686         /* Thread. */
3687         thread_ip_inc(p);
3688 }
3689
3690 static inline void
3691 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3692 {
3693         struct thread *t = &p->threads[p->thread_id];
3694         struct instruction *ip = t->ip;
3695
3696         /* Structs. */
3697         __instr_alu_xor_mh_exec(p, t, ip);
3698
3699         /* Thread. */
3700         thread_ip_inc(p);
3701 }
3702
3703 static inline void
3704 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3705 {
3706         struct thread *t = &p->threads[p->thread_id];
3707         struct instruction *ip = t->ip;
3708
3709         /* Structs. */
3710         __instr_alu_xor_hm_exec(p, t, ip);
3711
3712         /* Thread. */
3713         thread_ip_inc(p);
3714 }
3715
3716 static inline void
3717 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3718 {
3719         struct thread *t = &p->threads[p->thread_id];
3720         struct instruction *ip = t->ip;
3721
3722         /* Structs. */
3723         __instr_alu_xor_hh_exec(p, t, ip);
3724
3725         /* Thread. */
3726         thread_ip_inc(p);
3727 }
3728
3729 static inline void
3730 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3731 {
3732         struct thread *t = &p->threads[p->thread_id];
3733         struct instruction *ip = t->ip;
3734
3735         /* Structs. */
3736         __instr_alu_xor_i_exec(p, t, ip);
3737
3738         /* Thread. */
3739         thread_ip_inc(p);
3740 }
3741
3742 static inline void
3743 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3744 {
3745         struct thread *t = &p->threads[p->thread_id];
3746         struct instruction *ip = t->ip;
3747
3748         /* Structs. */
3749         __instr_alu_ckadd_field_exec(p, t, ip);
3750
3751         /* Thread. */
3752         thread_ip_inc(p);
3753 }
3754
3755 static inline void
3756 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3757 {
3758         struct thread *t = &p->threads[p->thread_id];
3759         struct instruction *ip = t->ip;
3760
3761         /* Structs. */
3762         __instr_alu_cksub_field_exec(p, t, ip);
3763
3764         /* Thread. */
3765         thread_ip_inc(p);
3766 }
3767
3768 static inline void
3769 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3770 {
3771         struct thread *t = &p->threads[p->thread_id];
3772         struct instruction *ip = t->ip;
3773
3774         /* Structs. */
3775         __instr_alu_ckadd_struct20_exec(p, t, ip);
3776
3777         /* Thread. */
3778         thread_ip_inc(p);
3779 }
3780
3781 static inline void
3782 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3783 {
3784         struct thread *t = &p->threads[p->thread_id];
3785         struct instruction *ip = t->ip;
3786
3787         /* Structs. */
3788         __instr_alu_ckadd_struct_exec(p, t, ip);
3789
3790         /* Thread. */
3791         thread_ip_inc(p);
3792 }
3793
3794 /*
3795  * Register array.
3796  */
3797 static struct regarray *
3798 regarray_find(struct rte_swx_pipeline *p, const char *name);
3799
3800 static int
3801 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3802                       struct action *action,
3803                       char **tokens,
3804                       int n_tokens,
3805                       struct instruction *instr,
3806                       struct instruction_data *data __rte_unused)
3807 {
3808         char *regarray = tokens[1], *idx = tokens[2];
3809         struct regarray *r;
3810         struct field *fidx;
3811         uint32_t idx_struct_id, idx_val;
3812
3813         CHECK(n_tokens == 3, EINVAL);
3814
3815         r = regarray_find(p, regarray);
3816         CHECK(r, EINVAL);
3817
3818         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3819         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3820         if (fidx) {
3821                 CHECK(!fidx->var_size, EINVAL);
3822
3823                 instr->type = INSTR_REGPREFETCH_RM;
3824                 if (idx[0] == 'h')
3825                         instr->type = INSTR_REGPREFETCH_RH;
3826
3827                 instr->regarray.regarray_id = r->id;
3828                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3829                 instr->regarray.idx.n_bits = fidx->n_bits;
3830                 instr->regarray.idx.offset = fidx->offset / 8;
3831                 instr->regarray.dstsrc_val = 0; /* Unused. */
3832                 return 0;
3833         }
3834
3835         /* REGPREFETCH_RI. */
3836         idx_val = strtoul(idx, &idx, 0);
3837         CHECK(!idx[0], EINVAL);
3838
3839         instr->type = INSTR_REGPREFETCH_RI;
3840         instr->regarray.regarray_id = r->id;
3841         instr->regarray.idx_val = idx_val;
3842         instr->regarray.dstsrc_val = 0; /* Unused. */
3843         return 0;
3844 }
3845
3846 static int
3847 instr_regrd_translate(struct rte_swx_pipeline *p,
3848                       struct action *action,
3849                       char **tokens,
3850                       int n_tokens,
3851                       struct instruction *instr,
3852                       struct instruction_data *data __rte_unused)
3853 {
3854         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3855         struct regarray *r;
3856         struct field *fdst, *fidx;
3857         uint32_t dst_struct_id, idx_struct_id, idx_val;
3858
3859         CHECK(n_tokens == 4, EINVAL);
3860
3861         r = regarray_find(p, regarray);
3862         CHECK(r, EINVAL);
3863
3864         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3865         CHECK(fdst, EINVAL);
3866         CHECK(!fdst->var_size, EINVAL);
3867
3868         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3869         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3870         if (fidx) {
3871                 CHECK(!fidx->var_size, EINVAL);
3872
3873                 instr->type = INSTR_REGRD_MRM;
3874                 if (dst[0] == 'h' && idx[0] != 'h')
3875                         instr->type = INSTR_REGRD_HRM;
3876                 if (dst[0] != 'h' && idx[0] == 'h')
3877                         instr->type = INSTR_REGRD_MRH;
3878                 if (dst[0] == 'h' && idx[0] == 'h')
3879                         instr->type = INSTR_REGRD_HRH;
3880
3881                 instr->regarray.regarray_id = r->id;
3882                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3883                 instr->regarray.idx.n_bits = fidx->n_bits;
3884                 instr->regarray.idx.offset = fidx->offset / 8;
3885                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3886                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3887                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3888                 return 0;
3889         }
3890
3891         /* REGRD_MRI, REGRD_HRI. */
3892         idx_val = strtoul(idx, &idx, 0);
3893         CHECK(!idx[0], EINVAL);
3894
3895         instr->type = INSTR_REGRD_MRI;
3896         if (dst[0] == 'h')
3897                 instr->type = INSTR_REGRD_HRI;
3898
3899         instr->regarray.regarray_id = r->id;
3900         instr->regarray.idx_val = idx_val;
3901         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3902         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3903         instr->regarray.dstsrc.offset = fdst->offset / 8;
3904         return 0;
3905 }
3906
3907 static int
3908 instr_regwr_translate(struct rte_swx_pipeline *p,
3909                       struct action *action,
3910                       char **tokens,
3911                       int n_tokens,
3912                       struct instruction *instr,
3913                       struct instruction_data *data __rte_unused)
3914 {
3915         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3916         struct regarray *r;
3917         struct field *fidx, *fsrc;
3918         uint64_t src_val;
3919         uint32_t idx_struct_id, idx_val, src_struct_id;
3920
3921         CHECK(n_tokens == 4, EINVAL);
3922
3923         r = regarray_find(p, regarray);
3924         CHECK(r, EINVAL);
3925
3926         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3927         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3928         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3929         if (fidx && fsrc) {
3930                 CHECK(!fidx->var_size, EINVAL);
3931                 CHECK(!fsrc->var_size, EINVAL);
3932
3933                 instr->type = INSTR_REGWR_RMM;
3934                 if (idx[0] == 'h' && src[0] != 'h')
3935                         instr->type = INSTR_REGWR_RHM;
3936                 if (idx[0] != 'h' && src[0] == 'h')
3937                         instr->type = INSTR_REGWR_RMH;
3938                 if (idx[0] == 'h' && src[0] == 'h')
3939                         instr->type = INSTR_REGWR_RHH;
3940
3941                 instr->regarray.regarray_id = r->id;
3942                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3943                 instr->regarray.idx.n_bits = fidx->n_bits;
3944                 instr->regarray.idx.offset = fidx->offset / 8;
3945                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3946                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3947                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3948                 return 0;
3949         }
3950
3951         /* REGWR_RHI, REGWR_RMI. */
3952         if (fidx && !fsrc) {
3953                 CHECK(!fidx->var_size, EINVAL);
3954
3955                 src_val = strtoull(src, &src, 0);
3956                 CHECK(!src[0], EINVAL);
3957
3958                 instr->type = INSTR_REGWR_RMI;
3959                 if (idx[0] == 'h')
3960                         instr->type = INSTR_REGWR_RHI;
3961
3962                 instr->regarray.regarray_id = r->id;
3963                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3964                 instr->regarray.idx.n_bits = fidx->n_bits;
3965                 instr->regarray.idx.offset = fidx->offset / 8;
3966                 instr->regarray.dstsrc_val = src_val;
3967                 return 0;
3968         }
3969
3970         /* REGWR_RIH, REGWR_RIM. */
3971         if (!fidx && fsrc) {
3972                 idx_val = strtoul(idx, &idx, 0);
3973                 CHECK(!idx[0], EINVAL);
3974
3975                 CHECK(!fsrc->var_size, EINVAL);
3976
3977                 instr->type = INSTR_REGWR_RIM;
3978                 if (src[0] == 'h')
3979                         instr->type = INSTR_REGWR_RIH;
3980
3981                 instr->regarray.regarray_id = r->id;
3982                 instr->regarray.idx_val = idx_val;
3983                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3984                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3985                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3986                 return 0;
3987         }
3988
3989         /* REGWR_RII. */
3990         src_val = strtoull(src, &src, 0);
3991         CHECK(!src[0], EINVAL);
3992
3993         idx_val = strtoul(idx, &idx, 0);
3994         CHECK(!idx[0], EINVAL);
3995
3996         instr->type = INSTR_REGWR_RII;
3997         instr->regarray.idx_val = idx_val;
3998         instr->regarray.dstsrc_val = src_val;
3999
4000         return 0;
4001 }
4002
4003 static int
4004 instr_regadd_translate(struct rte_swx_pipeline *p,
4005                        struct action *action,
4006                        char **tokens,
4007                        int n_tokens,
4008                        struct instruction *instr,
4009                        struct instruction_data *data __rte_unused)
4010 {
4011         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4012         struct regarray *r;
4013         struct field *fidx, *fsrc;
4014         uint64_t src_val;
4015         uint32_t idx_struct_id, idx_val, src_struct_id;
4016
4017         CHECK(n_tokens == 4, EINVAL);
4018
4019         r = regarray_find(p, regarray);
4020         CHECK(r, EINVAL);
4021
4022         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4023         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4024         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4025         if (fidx && fsrc) {
4026                 CHECK(!fidx->var_size, EINVAL);
4027                 CHECK(!fsrc->var_size, EINVAL);
4028
4029                 instr->type = INSTR_REGADD_RMM;
4030                 if (idx[0] == 'h' && src[0] != 'h')
4031                         instr->type = INSTR_REGADD_RHM;
4032                 if (idx[0] != 'h' && src[0] == 'h')
4033                         instr->type = INSTR_REGADD_RMH;
4034                 if (idx[0] == 'h' && src[0] == 'h')
4035                         instr->type = INSTR_REGADD_RHH;
4036
4037                 instr->regarray.regarray_id = r->id;
4038                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4039                 instr->regarray.idx.n_bits = fidx->n_bits;
4040                 instr->regarray.idx.offset = fidx->offset / 8;
4041                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4042                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4043                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4044                 return 0;
4045         }
4046
4047         /* REGADD_RHI, REGADD_RMI. */
4048         if (fidx && !fsrc) {
4049                 CHECK(!fidx->var_size, EINVAL);
4050
4051                 src_val = strtoull(src, &src, 0);
4052                 CHECK(!src[0], EINVAL);
4053
4054                 instr->type = INSTR_REGADD_RMI;
4055                 if (idx[0] == 'h')
4056                         instr->type = INSTR_REGADD_RHI;
4057
4058                 instr->regarray.regarray_id = r->id;
4059                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4060                 instr->regarray.idx.n_bits = fidx->n_bits;
4061                 instr->regarray.idx.offset = fidx->offset / 8;
4062                 instr->regarray.dstsrc_val = src_val;
4063                 return 0;
4064         }
4065
4066         /* REGADD_RIH, REGADD_RIM. */
4067         if (!fidx && fsrc) {
4068                 idx_val = strtoul(idx, &idx, 0);
4069                 CHECK(!idx[0], EINVAL);
4070
4071                 CHECK(!fsrc->var_size, EINVAL);
4072
4073                 instr->type = INSTR_REGADD_RIM;
4074                 if (src[0] == 'h')
4075                         instr->type = INSTR_REGADD_RIH;
4076
4077                 instr->regarray.regarray_id = r->id;
4078                 instr->regarray.idx_val = idx_val;
4079                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4080                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4081                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4082                 return 0;
4083         }
4084
4085         /* REGADD_RII. */
4086         src_val = strtoull(src, &src, 0);
4087         CHECK(!src[0], EINVAL);
4088
4089         idx_val = strtoul(idx, &idx, 0);
4090         CHECK(!idx[0], EINVAL);
4091
4092         instr->type = INSTR_REGADD_RII;
4093         instr->regarray.idx_val = idx_val;
4094         instr->regarray.dstsrc_val = src_val;
4095         return 0;
4096 }
4097
4098 static inline void
4099 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4100 {
4101         struct thread *t = &p->threads[p->thread_id];
4102         struct instruction *ip = t->ip;
4103
4104         /* Structs. */
4105         __instr_regprefetch_rh_exec(p, t, ip);
4106
4107         /* Thread. */
4108         thread_ip_inc(p);
4109 }
4110
4111 static inline void
4112 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4113 {
4114         struct thread *t = &p->threads[p->thread_id];
4115         struct instruction *ip = t->ip;
4116
4117         /* Structs. */
4118         __instr_regprefetch_rm_exec(p, t, ip);
4119
4120         /* Thread. */
4121         thread_ip_inc(p);
4122 }
4123
4124 static inline void
4125 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4126 {
4127         struct thread *t = &p->threads[p->thread_id];
4128         struct instruction *ip = t->ip;
4129
4130         /* Structs. */
4131         __instr_regprefetch_ri_exec(p, t, ip);
4132
4133         /* Thread. */
4134         thread_ip_inc(p);
4135 }
4136
4137 static inline void
4138 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4139 {
4140         struct thread *t = &p->threads[p->thread_id];
4141         struct instruction *ip = t->ip;
4142
4143         /* Structs. */
4144         __instr_regrd_hrh_exec(p, t, ip);
4145
4146         /* Thread. */
4147         thread_ip_inc(p);
4148 }
4149
4150 static inline void
4151 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4152 {
4153         struct thread *t = &p->threads[p->thread_id];
4154         struct instruction *ip = t->ip;
4155
4156         /* Structs. */
4157         __instr_regrd_hrm_exec(p, t, ip);
4158
4159         /* Thread. */
4160         thread_ip_inc(p);
4161 }
4162
4163 static inline void
4164 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4165 {
4166         struct thread *t = &p->threads[p->thread_id];
4167         struct instruction *ip = t->ip;
4168
4169         /* Structs. */
4170         __instr_regrd_mrh_exec(p, t, ip);
4171
4172         /* Thread. */
4173         thread_ip_inc(p);
4174 }
4175
4176 static inline void
4177 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4178 {
4179         struct thread *t = &p->threads[p->thread_id];
4180         struct instruction *ip = t->ip;
4181
4182         /* Structs. */
4183         __instr_regrd_mrm_exec(p, t, ip);
4184
4185         /* Thread. */
4186         thread_ip_inc(p);
4187 }
4188
4189 static inline void
4190 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4191 {
4192         struct thread *t = &p->threads[p->thread_id];
4193         struct instruction *ip = t->ip;
4194
4195         /* Structs. */
4196         __instr_regrd_hri_exec(p, t, ip);
4197
4198         /* Thread. */
4199         thread_ip_inc(p);
4200 }
4201
4202 static inline void
4203 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4204 {
4205         struct thread *t = &p->threads[p->thread_id];
4206         struct instruction *ip = t->ip;
4207
4208         /* Structs. */
4209         __instr_regrd_mri_exec(p, t, ip);
4210
4211         /* Thread. */
4212         thread_ip_inc(p);
4213 }
4214
4215 static inline void
4216 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4217 {
4218         struct thread *t = &p->threads[p->thread_id];
4219         struct instruction *ip = t->ip;
4220
4221         /* Structs. */
4222         __instr_regwr_rhh_exec(p, t, ip);
4223
4224         /* Thread. */
4225         thread_ip_inc(p);
4226 }
4227
4228 static inline void
4229 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4230 {
4231         struct thread *t = &p->threads[p->thread_id];
4232         struct instruction *ip = t->ip;
4233
4234         /* Structs. */
4235         __instr_regwr_rhm_exec(p, t, ip);
4236
4237         /* Thread. */
4238         thread_ip_inc(p);
4239 }
4240
4241 static inline void
4242 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4243 {
4244         struct thread *t = &p->threads[p->thread_id];
4245         struct instruction *ip = t->ip;
4246
4247         /* Structs. */
4248         __instr_regwr_rmh_exec(p, t, ip);
4249
4250         /* Thread. */
4251         thread_ip_inc(p);
4252 }
4253
4254 static inline void
4255 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4256 {
4257         struct thread *t = &p->threads[p->thread_id];
4258         struct instruction *ip = t->ip;
4259
4260         /* Structs. */
4261         __instr_regwr_rmm_exec(p, t, ip);
4262
4263         /* Thread. */
4264         thread_ip_inc(p);
4265 }
4266
4267 static inline void
4268 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4269 {
4270         struct thread *t = &p->threads[p->thread_id];
4271         struct instruction *ip = t->ip;
4272
4273         /* Structs. */
4274         __instr_regwr_rhi_exec(p, t, ip);
4275
4276         /* Thread. */
4277         thread_ip_inc(p);
4278 }
4279
4280 static inline void
4281 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4282 {
4283         struct thread *t = &p->threads[p->thread_id];
4284         struct instruction *ip = t->ip;
4285
4286         /* Structs. */
4287         __instr_regwr_rmi_exec(p, t, ip);
4288
4289         /* Thread. */
4290         thread_ip_inc(p);
4291 }
4292
4293 static inline void
4294 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4295 {
4296         struct thread *t = &p->threads[p->thread_id];
4297         struct instruction *ip = t->ip;
4298
4299         /* Structs. */
4300         __instr_regwr_rih_exec(p, t, ip);
4301
4302         /* Thread. */
4303         thread_ip_inc(p);
4304 }
4305
4306 static inline void
4307 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4308 {
4309         struct thread *t = &p->threads[p->thread_id];
4310         struct instruction *ip = t->ip;
4311
4312         /* Structs. */
4313         __instr_regwr_rim_exec(p, t, ip);
4314
4315         /* Thread. */
4316         thread_ip_inc(p);
4317 }
4318
4319 static inline void
4320 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4321 {
4322         struct thread *t = &p->threads[p->thread_id];
4323         struct instruction *ip = t->ip;
4324
4325         /* Structs. */
4326         __instr_regwr_rii_exec(p, t, ip);
4327
4328         /* Thread. */
4329         thread_ip_inc(p);
4330 }
4331
4332 static inline void
4333 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4334 {
4335         struct thread *t = &p->threads[p->thread_id];
4336         struct instruction *ip = t->ip;
4337
4338         /* Structs. */
4339         __instr_regadd_rhh_exec(p, t, ip);
4340
4341         /* Thread. */
4342         thread_ip_inc(p);
4343 }
4344
4345 static inline void
4346 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4347 {
4348         struct thread *t = &p->threads[p->thread_id];
4349         struct instruction *ip = t->ip;
4350
4351         /* Structs. */
4352         __instr_regadd_rhm_exec(p, t, ip);
4353
4354         /* Thread. */
4355         thread_ip_inc(p);
4356 }
4357
4358 static inline void
4359 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4360 {
4361         struct thread *t = &p->threads[p->thread_id];
4362         struct instruction *ip = t->ip;
4363
4364         /* Structs. */
4365         __instr_regadd_rmh_exec(p, t, ip);
4366
4367         /* Thread. */
4368         thread_ip_inc(p);
4369 }
4370
4371 static inline void
4372 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4373 {
4374         struct thread *t = &p->threads[p->thread_id];
4375         struct instruction *ip = t->ip;
4376
4377         /* Structs. */
4378         __instr_regadd_rmm_exec(p, t, ip);
4379
4380         /* Thread. */
4381         thread_ip_inc(p);
4382 }
4383
4384 static inline void
4385 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4386 {
4387         struct thread *t = &p->threads[p->thread_id];
4388         struct instruction *ip = t->ip;
4389
4390         /* Structs. */
4391         __instr_regadd_rhi_exec(p, t, ip);
4392
4393         /* Thread. */
4394         thread_ip_inc(p);
4395 }
4396
4397 static inline void
4398 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4399 {
4400         struct thread *t = &p->threads[p->thread_id];
4401         struct instruction *ip = t->ip;
4402
4403         /* Structs. */
4404         __instr_regadd_rmi_exec(p, t, ip);
4405
4406         /* Thread. */
4407         thread_ip_inc(p);
4408 }
4409
4410 static inline void
4411 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4412 {
4413         struct thread *t = &p->threads[p->thread_id];
4414         struct instruction *ip = t->ip;
4415
4416         /* Structs. */
4417         __instr_regadd_rih_exec(p, t, ip);
4418
4419         /* Thread. */
4420         thread_ip_inc(p);
4421 }
4422
4423 static inline void
4424 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4425 {
4426         struct thread *t = &p->threads[p->thread_id];
4427         struct instruction *ip = t->ip;
4428
4429         /* Structs. */
4430         __instr_regadd_rim_exec(p, t, ip);
4431
4432         /* Thread. */
4433         thread_ip_inc(p);
4434 }
4435
4436 static inline void
4437 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4438 {
4439         struct thread *t = &p->threads[p->thread_id];
4440         struct instruction *ip = t->ip;
4441
4442         /* Structs. */
4443         __instr_regadd_rii_exec(p, t, ip);
4444
4445         /* Thread. */
4446         thread_ip_inc(p);
4447 }
4448
4449 /*
4450  * metarray.
4451  */
4452 static struct metarray *
4453 metarray_find(struct rte_swx_pipeline *p, const char *name);
4454
4455 static int
4456 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4457                             struct action *action,
4458                             char **tokens,
4459                             int n_tokens,
4460                             struct instruction *instr,
4461                             struct instruction_data *data __rte_unused)
4462 {
4463         char *metarray = tokens[1], *idx = tokens[2];
4464         struct metarray *m;
4465         struct field *fidx;
4466         uint32_t idx_struct_id, idx_val;
4467
4468         CHECK(n_tokens == 3, EINVAL);
4469
4470         m = metarray_find(p, metarray);
4471         CHECK(m, EINVAL);
4472
4473         /* METPREFETCH_H, METPREFETCH_M. */
4474         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4475         if (fidx) {
4476                 CHECK(!fidx->var_size, EINVAL);
4477
4478                 instr->type = INSTR_METPREFETCH_M;
4479                 if (idx[0] == 'h')
4480                         instr->type = INSTR_METPREFETCH_H;
4481
4482                 instr->meter.metarray_id = m->id;
4483                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4484                 instr->meter.idx.n_bits = fidx->n_bits;
4485                 instr->meter.idx.offset = fidx->offset / 8;
4486                 return 0;
4487         }
4488
4489         /* METPREFETCH_I. */
4490         idx_val = strtoul(idx, &idx, 0);
4491         CHECK(!idx[0], EINVAL);
4492
4493         instr->type = INSTR_METPREFETCH_I;
4494         instr->meter.metarray_id = m->id;
4495         instr->meter.idx_val = idx_val;
4496         return 0;
4497 }
4498
4499 static int
4500 instr_meter_translate(struct rte_swx_pipeline *p,
4501                       struct action *action,
4502                       char **tokens,
4503                       int n_tokens,
4504                       struct instruction *instr,
4505                       struct instruction_data *data __rte_unused)
4506 {
4507         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4508         char *color_in = tokens[4], *color_out = tokens[5];
4509         struct metarray *m;
4510         struct field *fidx, *flength, *fcin, *fcout;
4511         uint32_t idx_struct_id, length_struct_id;
4512         uint32_t color_in_struct_id, color_out_struct_id;
4513
4514         CHECK(n_tokens == 6, EINVAL);
4515
4516         m = metarray_find(p, metarray);
4517         CHECK(m, EINVAL);
4518
4519         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4520
4521         flength = struct_field_parse(p, action, length, &length_struct_id);
4522         CHECK(flength, EINVAL);
4523         CHECK(!flength->var_size, EINVAL);
4524
4525         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4526
4527         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4528         CHECK(fcout, EINVAL);
4529         CHECK(!fcout->var_size, EINVAL);
4530
4531         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4532         if (fidx && fcin) {
4533                 CHECK(!fidx->var_size, EINVAL);
4534                 CHECK(!fcin->var_size, EINVAL);
4535
4536                 instr->type = INSTR_METER_MMM;
4537                 if (idx[0] == 'h' && length[0] == 'h')
4538                         instr->type = INSTR_METER_HHM;
4539                 if (idx[0] == 'h' && length[0] != 'h')
4540                         instr->type = INSTR_METER_HMM;
4541                 if (idx[0] != 'h' && length[0] == 'h')
4542                         instr->type = INSTR_METER_MHM;
4543
4544                 instr->meter.metarray_id = m->id;
4545
4546                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4547                 instr->meter.idx.n_bits = fidx->n_bits;
4548                 instr->meter.idx.offset = fidx->offset / 8;
4549
4550                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4551                 instr->meter.length.n_bits = flength->n_bits;
4552                 instr->meter.length.offset = flength->offset / 8;
4553
4554                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4555                 instr->meter.color_in.n_bits = fcin->n_bits;
4556                 instr->meter.color_in.offset = fcin->offset / 8;
4557
4558                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4559                 instr->meter.color_out.n_bits = fcout->n_bits;
4560                 instr->meter.color_out.offset = fcout->offset / 8;
4561         }
4562
4563         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4564         if (fidx && !fcin) {
4565                 uint32_t color_in_val;
4566
4567                 CHECK(!fidx->var_size, EINVAL);
4568
4569                 color_in_val = strtoul(color_in, &color_in, 0);
4570                 CHECK(!color_in[0], EINVAL);
4571
4572                 instr->type = INSTR_METER_MMI;
4573                 if (idx[0] == 'h' && length[0] == 'h')
4574                         instr->type = INSTR_METER_HHI;
4575                 if (idx[0] == 'h' && length[0] != 'h')
4576                         instr->type = INSTR_METER_HMI;
4577                 if (idx[0] != 'h' && length[0] == 'h')
4578                         instr->type = INSTR_METER_MHI;
4579
4580                 instr->meter.metarray_id = m->id;
4581
4582                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4583                 instr->meter.idx.n_bits = fidx->n_bits;
4584                 instr->meter.idx.offset = fidx->offset / 8;
4585
4586                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4587                 instr->meter.length.n_bits = flength->n_bits;
4588                 instr->meter.length.offset = flength->offset / 8;
4589
4590                 instr->meter.color_in_val = color_in_val;
4591
4592                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4593                 instr->meter.color_out.n_bits = fcout->n_bits;
4594                 instr->meter.color_out.offset = fcout->offset / 8;
4595         }
4596
4597         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4598         if (!fidx && fcin) {
4599                 uint32_t idx_val;
4600
4601                 idx_val = strtoul(idx, &idx, 0);
4602                 CHECK(!idx[0], EINVAL);
4603
4604                 CHECK(!fcin->var_size, EINVAL);
4605
4606                 instr->type = INSTR_METER_IMM;
4607                 if (length[0] == 'h')
4608                         instr->type = INSTR_METER_IHM;
4609
4610                 instr->meter.metarray_id = m->id;
4611
4612                 instr->meter.idx_val = idx_val;
4613
4614                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4615                 instr->meter.length.n_bits = flength->n_bits;
4616                 instr->meter.length.offset = flength->offset / 8;
4617
4618                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4619                 instr->meter.color_in.n_bits = fcin->n_bits;
4620                 instr->meter.color_in.offset = fcin->offset / 8;
4621
4622                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4623                 instr->meter.color_out.n_bits = fcout->n_bits;
4624                 instr->meter.color_out.offset = fcout->offset / 8;
4625         }
4626
4627         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4628         if (!fidx && !fcin) {
4629                 uint32_t idx_val, color_in_val;
4630
4631                 idx_val = strtoul(idx, &idx, 0);
4632                 CHECK(!idx[0], EINVAL);
4633
4634                 color_in_val = strtoul(color_in, &color_in, 0);
4635                 CHECK(!color_in[0], EINVAL);
4636
4637                 instr->type = INSTR_METER_IMI;
4638                 if (length[0] == 'h')
4639                         instr->type = INSTR_METER_IHI;
4640
4641                 instr->meter.metarray_id = m->id;
4642
4643                 instr->meter.idx_val = idx_val;
4644
4645                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4646                 instr->meter.length.n_bits = flength->n_bits;
4647                 instr->meter.length.offset = flength->offset / 8;
4648
4649                 instr->meter.color_in_val = color_in_val;
4650
4651                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4652                 instr->meter.color_out.n_bits = fcout->n_bits;
4653                 instr->meter.color_out.offset = fcout->offset / 8;
4654         }
4655
4656         return 0;
4657 }
4658
4659 static inline void
4660 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4661 {
4662         struct thread *t = &p->threads[p->thread_id];
4663         struct instruction *ip = t->ip;
4664
4665         /* Structs. */
4666         __instr_metprefetch_h_exec(p, t, ip);
4667
4668         /* Thread. */
4669         thread_ip_inc(p);
4670 }
4671
4672 static inline void
4673 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4674 {
4675         struct thread *t = &p->threads[p->thread_id];
4676         struct instruction *ip = t->ip;
4677
4678         /* Structs. */
4679         __instr_metprefetch_m_exec(p, t, ip);
4680
4681         /* Thread. */
4682         thread_ip_inc(p);
4683 }
4684
4685 static inline void
4686 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4687 {
4688         struct thread *t = &p->threads[p->thread_id];
4689         struct instruction *ip = t->ip;
4690
4691         /* Structs. */
4692         __instr_metprefetch_i_exec(p, t, ip);
4693
4694         /* Thread. */
4695         thread_ip_inc(p);
4696 }
4697
4698 static inline void
4699 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4700 {
4701         struct thread *t = &p->threads[p->thread_id];
4702         struct instruction *ip = t->ip;
4703
4704         /* Structs. */
4705         __instr_meter_hhm_exec(p, t, ip);
4706
4707         /* Thread. */
4708         thread_ip_inc(p);
4709 }
4710
4711 static inline void
4712 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4713 {
4714         struct thread *t = &p->threads[p->thread_id];
4715         struct instruction *ip = t->ip;
4716
4717         /* Structs. */
4718         __instr_meter_hhi_exec(p, t, ip);
4719
4720         /* Thread. */
4721         thread_ip_inc(p);
4722 }
4723
4724 static inline void
4725 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4726 {
4727         struct thread *t = &p->threads[p->thread_id];
4728         struct instruction *ip = t->ip;
4729
4730         /* Structs. */
4731         __instr_meter_hmm_exec(p, t, ip);
4732
4733         /* Thread. */
4734         thread_ip_inc(p);
4735 }
4736
4737 static inline void
4738 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4739 {
4740         struct thread *t = &p->threads[p->thread_id];
4741         struct instruction *ip = t->ip;
4742
4743         /* Structs. */
4744         __instr_meter_hmi_exec(p, t, ip);
4745
4746         /* Thread. */
4747         thread_ip_inc(p);
4748 }
4749
4750 static inline void
4751 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4752 {
4753         struct thread *t = &p->threads[p->thread_id];
4754         struct instruction *ip = t->ip;
4755
4756         /* Structs. */
4757         __instr_meter_mhm_exec(p, t, ip);
4758
4759         /* Thread. */
4760         thread_ip_inc(p);
4761 }
4762
4763 static inline void
4764 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4765 {
4766         struct thread *t = &p->threads[p->thread_id];
4767         struct instruction *ip = t->ip;
4768
4769         /* Structs. */
4770         __instr_meter_mhi_exec(p, t, ip);
4771
4772         /* Thread. */
4773         thread_ip_inc(p);
4774 }
4775
4776 static inline void
4777 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4778 {
4779         struct thread *t = &p->threads[p->thread_id];
4780         struct instruction *ip = t->ip;
4781
4782         /* Structs. */
4783         __instr_meter_mmm_exec(p, t, ip);
4784
4785         /* Thread. */
4786         thread_ip_inc(p);
4787 }
4788
4789 static inline void
4790 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4791 {
4792         struct thread *t = &p->threads[p->thread_id];
4793         struct instruction *ip = t->ip;
4794
4795         /* Structs. */
4796         __instr_meter_mmi_exec(p, t, ip);
4797
4798         /* Thread. */
4799         thread_ip_inc(p);
4800 }
4801
4802 static inline void
4803 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4804 {
4805         struct thread *t = &p->threads[p->thread_id];
4806         struct instruction *ip = t->ip;
4807
4808         /* Structs. */
4809         __instr_meter_ihm_exec(p, t, ip);
4810
4811         /* Thread. */
4812         thread_ip_inc(p);
4813 }
4814
4815 static inline void
4816 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4817 {
4818         struct thread *t = &p->threads[p->thread_id];
4819         struct instruction *ip = t->ip;
4820
4821         /* Structs. */
4822         __instr_meter_ihi_exec(p, t, ip);
4823
4824         /* Thread. */
4825         thread_ip_inc(p);
4826 }
4827
4828 static inline void
4829 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4830 {
4831         struct thread *t = &p->threads[p->thread_id];
4832         struct instruction *ip = t->ip;
4833
4834         /* Structs. */
4835         __instr_meter_imm_exec(p, t, ip);
4836
4837         /* Thread. */
4838         thread_ip_inc(p);
4839 }
4840
4841 static inline void
4842 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4843 {
4844         struct thread *t = &p->threads[p->thread_id];
4845         struct instruction *ip = t->ip;
4846
4847         /* Structs. */
4848         __instr_meter_imi_exec(p, t, ip);
4849
4850         /* Thread. */
4851         thread_ip_inc(p);
4852 }
4853
4854 /*
4855  * jmp.
4856  */
4857 static int
4858 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4859                     struct action *action __rte_unused,
4860                     char **tokens,
4861                     int n_tokens,
4862                     struct instruction *instr,
4863                     struct instruction_data *data)
4864 {
4865         CHECK(n_tokens == 2, EINVAL);
4866
4867         strcpy(data->jmp_label, tokens[1]);
4868
4869         instr->type = INSTR_JMP;
4870         instr->jmp.ip = NULL; /* Resolved later. */
4871         return 0;
4872 }
4873
4874 static int
4875 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4876                           struct action *action __rte_unused,
4877                           char **tokens,
4878                           int n_tokens,
4879                           struct instruction *instr,
4880                           struct instruction_data *data)
4881 {
4882         struct header *h;
4883
4884         CHECK(n_tokens == 3, EINVAL);
4885
4886         strcpy(data->jmp_label, tokens[1]);
4887
4888         h = header_parse(p, tokens[2]);
4889         CHECK(h, EINVAL);
4890
4891         instr->type = INSTR_JMP_VALID;
4892         instr->jmp.ip = NULL; /* Resolved later. */
4893         instr->jmp.header_id = h->id;
4894         return 0;
4895 }
4896
4897 static int
4898 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4899                             struct action *action __rte_unused,
4900                             char **tokens,
4901                             int n_tokens,
4902                             struct instruction *instr,
4903                             struct instruction_data *data)
4904 {
4905         struct header *h;
4906
4907         CHECK(n_tokens == 3, EINVAL);
4908
4909         strcpy(data->jmp_label, tokens[1]);
4910
4911         h = header_parse(p, tokens[2]);
4912         CHECK(h, EINVAL);
4913
4914         instr->type = INSTR_JMP_INVALID;
4915         instr->jmp.ip = NULL; /* Resolved later. */
4916         instr->jmp.header_id = h->id;
4917         return 0;
4918 }
4919
4920 static int
4921 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4922                         struct action *action,
4923                         char **tokens,
4924                         int n_tokens,
4925                         struct instruction *instr,
4926                         struct instruction_data *data)
4927 {
4928         CHECK(!action, EINVAL);
4929         CHECK(n_tokens == 2, EINVAL);
4930
4931         strcpy(data->jmp_label, tokens[1]);
4932
4933         instr->type = INSTR_JMP_HIT;
4934         instr->jmp.ip = NULL; /* Resolved later. */
4935         return 0;
4936 }
4937
4938 static int
4939 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4940                          struct action *action,
4941                          char **tokens,
4942                          int n_tokens,
4943                          struct instruction *instr,
4944                          struct instruction_data *data)
4945 {
4946         CHECK(!action, EINVAL);
4947         CHECK(n_tokens == 2, EINVAL);
4948
4949         strcpy(data->jmp_label, tokens[1]);
4950
4951         instr->type = INSTR_JMP_MISS;
4952         instr->jmp.ip = NULL; /* Resolved later. */
4953         return 0;
4954 }
4955
4956 static int
4957 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4958                                struct action *action,
4959                                char **tokens,
4960                                int n_tokens,
4961                                struct instruction *instr,
4962                                struct instruction_data *data)
4963 {
4964         struct action *a;
4965
4966         CHECK(!action, EINVAL);
4967         CHECK(n_tokens == 3, EINVAL);
4968
4969         strcpy(data->jmp_label, tokens[1]);
4970
4971         a = action_find(p, tokens[2]);
4972         CHECK(a, EINVAL);
4973
4974         instr->type = INSTR_JMP_ACTION_HIT;
4975         instr->jmp.ip = NULL; /* Resolved later. */
4976         instr->jmp.action_id = a->id;
4977         return 0;
4978 }
4979
4980 static int
4981 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4982                                 struct action *action,
4983                                 char **tokens,
4984                                 int n_tokens,
4985                                 struct instruction *instr,
4986                                 struct instruction_data *data)
4987 {
4988         struct action *a;
4989
4990         CHECK(!action, EINVAL);
4991         CHECK(n_tokens == 3, EINVAL);
4992
4993         strcpy(data->jmp_label, tokens[1]);
4994
4995         a = action_find(p, tokens[2]);
4996         CHECK(a, EINVAL);
4997
4998         instr->type = INSTR_JMP_ACTION_MISS;
4999         instr->jmp.ip = NULL; /* Resolved later. */
5000         instr->jmp.action_id = a->id;
5001         return 0;
5002 }
5003
5004 static int
5005 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5006                        struct action *action,
5007                        char **tokens,
5008                        int n_tokens,
5009                        struct instruction *instr,
5010                        struct instruction_data *data)
5011 {
5012         char *a = tokens[2], *b = tokens[3];
5013         struct field *fa, *fb;
5014         uint64_t b_val;
5015         uint32_t a_struct_id, b_struct_id;
5016
5017         CHECK(n_tokens == 4, EINVAL);
5018
5019         strcpy(data->jmp_label, tokens[1]);
5020
5021         fa = struct_field_parse(p, action, a, &a_struct_id);
5022         CHECK(fa, EINVAL);
5023         CHECK(!fa->var_size, EINVAL);
5024
5025         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5026         fb = struct_field_parse(p, action, b, &b_struct_id);
5027         if (fb) {
5028                 CHECK(!fb->var_size, EINVAL);
5029
5030                 instr->type = INSTR_JMP_EQ;
5031                 if (a[0] != 'h' && b[0] == 'h')
5032                         instr->type = INSTR_JMP_EQ_MH;
5033                 if (a[0] == 'h' && b[0] != 'h')
5034                         instr->type = INSTR_JMP_EQ_HM;
5035                 if (a[0] == 'h' && b[0] == 'h')
5036                         instr->type = INSTR_JMP_EQ_HH;
5037                 instr->jmp.ip = NULL; /* Resolved later. */
5038
5039                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5040                 instr->jmp.a.n_bits = fa->n_bits;
5041                 instr->jmp.a.offset = fa->offset / 8;
5042                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5043                 instr->jmp.b.n_bits = fb->n_bits;
5044                 instr->jmp.b.offset = fb->offset / 8;
5045                 return 0;
5046         }
5047
5048         /* JMP_EQ_I. */
5049         b_val = strtoull(b, &b, 0);
5050         CHECK(!b[0], EINVAL);
5051
5052         if (a[0] == 'h')
5053                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5054
5055         instr->type = INSTR_JMP_EQ_I;
5056         instr->jmp.ip = NULL; /* Resolved later. */
5057         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5058         instr->jmp.a.n_bits = fa->n_bits;
5059         instr->jmp.a.offset = fa->offset / 8;
5060         instr->jmp.b_val = b_val;
5061         return 0;
5062 }
5063
5064 static int
5065 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5066                         struct action *action,
5067                         char **tokens,
5068                         int n_tokens,
5069                         struct instruction *instr,
5070                         struct instruction_data *data)
5071 {
5072         char *a = tokens[2], *b = tokens[3];
5073         struct field *fa, *fb;
5074         uint64_t b_val;
5075         uint32_t a_struct_id, b_struct_id;
5076
5077         CHECK(n_tokens == 4, EINVAL);
5078
5079         strcpy(data->jmp_label, tokens[1]);
5080
5081         fa = struct_field_parse(p, action, a, &a_struct_id);
5082         CHECK(fa, EINVAL);
5083         CHECK(!fa->var_size, EINVAL);
5084
5085         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5086         fb = struct_field_parse(p, action, b, &b_struct_id);
5087         if (fb) {
5088                 CHECK(!fb->var_size, EINVAL);
5089
5090                 instr->type = INSTR_JMP_NEQ;
5091                 if (a[0] != 'h' && b[0] == 'h')
5092                         instr->type = INSTR_JMP_NEQ_MH;
5093                 if (a[0] == 'h' && b[0] != 'h')
5094                         instr->type = INSTR_JMP_NEQ_HM;
5095                 if (a[0] == 'h' && b[0] == 'h')
5096                         instr->type = INSTR_JMP_NEQ_HH;
5097                 instr->jmp.ip = NULL; /* Resolved later. */
5098
5099                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5100                 instr->jmp.a.n_bits = fa->n_bits;
5101                 instr->jmp.a.offset = fa->offset / 8;
5102                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5103                 instr->jmp.b.n_bits = fb->n_bits;
5104                 instr->jmp.b.offset = fb->offset / 8;
5105                 return 0;
5106         }
5107
5108         /* JMP_NEQ_I. */
5109         b_val = strtoull(b, &b, 0);
5110         CHECK(!b[0], EINVAL);
5111
5112         if (a[0] == 'h')
5113                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5114
5115         instr->type = INSTR_JMP_NEQ_I;
5116         instr->jmp.ip = NULL; /* Resolved later. */
5117         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5118         instr->jmp.a.n_bits = fa->n_bits;
5119         instr->jmp.a.offset = fa->offset / 8;
5120         instr->jmp.b_val = b_val;
5121         return 0;
5122 }
5123
5124 static int
5125 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5126                        struct action *action,
5127                        char **tokens,
5128                        int n_tokens,
5129                        struct instruction *instr,
5130                        struct instruction_data *data)
5131 {
5132         char *a = tokens[2], *b = tokens[3];
5133         struct field *fa, *fb;
5134         uint64_t b_val;
5135         uint32_t a_struct_id, b_struct_id;
5136
5137         CHECK(n_tokens == 4, EINVAL);
5138
5139         strcpy(data->jmp_label, tokens[1]);
5140
5141         fa = struct_field_parse(p, action, a, &a_struct_id);
5142         CHECK(fa, EINVAL);
5143         CHECK(!fa->var_size, EINVAL);
5144
5145         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5146         fb = struct_field_parse(p, action, b, &b_struct_id);
5147         if (fb) {
5148                 CHECK(!fb->var_size, EINVAL);
5149
5150                 instr->type = INSTR_JMP_LT;
5151                 if (a[0] == 'h' && b[0] != 'h')
5152                         instr->type = INSTR_JMP_LT_HM;
5153                 if (a[0] != 'h' && b[0] == 'h')
5154                         instr->type = INSTR_JMP_LT_MH;
5155                 if (a[0] == 'h' && b[0] == 'h')
5156                         instr->type = INSTR_JMP_LT_HH;
5157                 instr->jmp.ip = NULL; /* Resolved later. */
5158
5159                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5160                 instr->jmp.a.n_bits = fa->n_bits;
5161                 instr->jmp.a.offset = fa->offset / 8;
5162                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5163                 instr->jmp.b.n_bits = fb->n_bits;
5164                 instr->jmp.b.offset = fb->offset / 8;
5165                 return 0;
5166         }
5167
5168         /* JMP_LT_MI, JMP_LT_HI. */
5169         b_val = strtoull(b, &b, 0);
5170         CHECK(!b[0], EINVAL);
5171
5172         instr->type = INSTR_JMP_LT_MI;
5173         if (a[0] == 'h')
5174                 instr->type = INSTR_JMP_LT_HI;
5175         instr->jmp.ip = NULL; /* Resolved later. */
5176
5177         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5178         instr->jmp.a.n_bits = fa->n_bits;
5179         instr->jmp.a.offset = fa->offset / 8;
5180         instr->jmp.b_val = b_val;
5181         return 0;
5182 }
5183
5184 static int
5185 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5186                        struct action *action,
5187                        char **tokens,
5188                        int n_tokens,
5189                        struct instruction *instr,
5190                        struct instruction_data *data)
5191 {
5192         char *a = tokens[2], *b = tokens[3];
5193         struct field *fa, *fb;
5194         uint64_t b_val;
5195         uint32_t a_struct_id, b_struct_id;
5196
5197         CHECK(n_tokens == 4, EINVAL);
5198
5199         strcpy(data->jmp_label, tokens[1]);
5200
5201         fa = struct_field_parse(p, action, a, &a_struct_id);
5202         CHECK(fa, EINVAL);
5203         CHECK(!fa->var_size, EINVAL);
5204
5205         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5206         fb = struct_field_parse(p, action, b, &b_struct_id);
5207         if (fb) {
5208                 CHECK(!fb->var_size, EINVAL);
5209
5210                 instr->type = INSTR_JMP_GT;
5211                 if (a[0] == 'h' && b[0] != 'h')
5212                         instr->type = INSTR_JMP_GT_HM;
5213                 if (a[0] != 'h' && b[0] == 'h')
5214                         instr->type = INSTR_JMP_GT_MH;
5215                 if (a[0] == 'h' && b[0] == 'h')
5216                         instr->type = INSTR_JMP_GT_HH;
5217                 instr->jmp.ip = NULL; /* Resolved later. */
5218
5219                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5220                 instr->jmp.a.n_bits = fa->n_bits;
5221                 instr->jmp.a.offset = fa->offset / 8;
5222                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5223                 instr->jmp.b.n_bits = fb->n_bits;
5224                 instr->jmp.b.offset = fb->offset / 8;
5225                 return 0;
5226         }
5227
5228         /* JMP_GT_MI, JMP_GT_HI. */
5229         b_val = strtoull(b, &b, 0);
5230         CHECK(!b[0], EINVAL);
5231
5232         instr->type = INSTR_JMP_GT_MI;
5233         if (a[0] == 'h')
5234                 instr->type = INSTR_JMP_GT_HI;
5235         instr->jmp.ip = NULL; /* Resolved later. */
5236
5237         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5238         instr->jmp.a.n_bits = fa->n_bits;
5239         instr->jmp.a.offset = fa->offset / 8;
5240         instr->jmp.b_val = b_val;
5241         return 0;
5242 }
5243
5244 static inline void
5245 instr_jmp_exec(struct rte_swx_pipeline *p)
5246 {
5247         struct thread *t = &p->threads[p->thread_id];
5248         struct instruction *ip = t->ip;
5249
5250         TRACE("[Thread %2u] jmp\n", p->thread_id);
5251
5252         thread_ip_set(t, ip->jmp.ip);
5253 }
5254
5255 static inline void
5256 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5257 {
5258         struct thread *t = &p->threads[p->thread_id];
5259         struct instruction *ip = t->ip;
5260         uint32_t header_id = ip->jmp.header_id;
5261
5262         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5263
5264         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5265 }
5266
5267 static inline void
5268 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5269 {
5270         struct thread *t = &p->threads[p->thread_id];
5271         struct instruction *ip = t->ip;
5272         uint32_t header_id = ip->jmp.header_id;
5273
5274         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5275
5276         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5277 }
5278
5279 static inline void
5280 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5281 {
5282         struct thread *t = &p->threads[p->thread_id];
5283         struct instruction *ip = t->ip;
5284         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5285
5286         TRACE("[Thread %2u] jmph\n", p->thread_id);
5287
5288         t->ip = ip_next[t->hit];
5289 }
5290
5291 static inline void
5292 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5293 {
5294         struct thread *t = &p->threads[p->thread_id];
5295         struct instruction *ip = t->ip;
5296         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5297
5298         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5299
5300         t->ip = ip_next[t->hit];
5301 }
5302
5303 static inline void
5304 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5305 {
5306         struct thread *t = &p->threads[p->thread_id];
5307         struct instruction *ip = t->ip;
5308
5309         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5310
5311         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5312 }
5313
5314 static inline void
5315 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5316 {
5317         struct thread *t = &p->threads[p->thread_id];
5318         struct instruction *ip = t->ip;
5319
5320         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5321
5322         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5323 }
5324
5325 static inline void
5326 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5327 {
5328         struct thread *t = &p->threads[p->thread_id];
5329         struct instruction *ip = t->ip;
5330
5331         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5332
5333         JMP_CMP(t, ip, ==);
5334 }
5335
5336 static inline void
5337 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5338 {
5339         struct thread *t = &p->threads[p->thread_id];
5340         struct instruction *ip = t->ip;
5341
5342         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5343
5344         JMP_CMP_MH(t, ip, ==);
5345 }
5346
5347 static inline void
5348 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5349 {
5350         struct thread *t = &p->threads[p->thread_id];
5351         struct instruction *ip = t->ip;
5352
5353         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5354
5355         JMP_CMP_HM(t, ip, ==);
5356 }
5357
5358 static inline void
5359 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5360 {
5361         struct thread *t = &p->threads[p->thread_id];
5362         struct instruction *ip = t->ip;
5363
5364         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5365
5366         JMP_CMP_HH_FAST(t, ip, ==);
5367 }
5368
5369 static inline void
5370 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5371 {
5372         struct thread *t = &p->threads[p->thread_id];
5373         struct instruction *ip = t->ip;
5374
5375         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5376
5377         JMP_CMP_I(t, ip, ==);
5378 }
5379
5380 static inline void
5381 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5382 {
5383         struct thread *t = &p->threads[p->thread_id];
5384         struct instruction *ip = t->ip;
5385
5386         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5387
5388         JMP_CMP(t, ip, !=);
5389 }
5390
5391 static inline void
5392 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5393 {
5394         struct thread *t = &p->threads[p->thread_id];
5395         struct instruction *ip = t->ip;
5396
5397         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5398
5399         JMP_CMP_MH(t, ip, !=);
5400 }
5401
5402 static inline void
5403 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5404 {
5405         struct thread *t = &p->threads[p->thread_id];
5406         struct instruction *ip = t->ip;
5407
5408         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5409
5410         JMP_CMP_HM(t, ip, !=);
5411 }
5412
5413 static inline void
5414 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5415 {
5416         struct thread *t = &p->threads[p->thread_id];
5417         struct instruction *ip = t->ip;
5418
5419         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5420
5421         JMP_CMP_HH_FAST(t, ip, !=);
5422 }
5423
5424 static inline void
5425 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5426 {
5427         struct thread *t = &p->threads[p->thread_id];
5428         struct instruction *ip = t->ip;
5429
5430         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5431
5432         JMP_CMP_I(t, ip, !=);
5433 }
5434
5435 static inline void
5436 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5437 {
5438         struct thread *t = &p->threads[p->thread_id];
5439         struct instruction *ip = t->ip;
5440
5441         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5442
5443         JMP_CMP(t, ip, <);
5444 }
5445
5446 static inline void
5447 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5448 {
5449         struct thread *t = &p->threads[p->thread_id];
5450         struct instruction *ip = t->ip;
5451
5452         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5453
5454         JMP_CMP_MH(t, ip, <);
5455 }
5456
5457 static inline void
5458 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5459 {
5460         struct thread *t = &p->threads[p->thread_id];
5461         struct instruction *ip = t->ip;
5462
5463         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5464
5465         JMP_CMP_HM(t, ip, <);
5466 }
5467
5468 static inline void
5469 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5470 {
5471         struct thread *t = &p->threads[p->thread_id];
5472         struct instruction *ip = t->ip;
5473
5474         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5475
5476         JMP_CMP_HH(t, ip, <);
5477 }
5478
5479 static inline void
5480 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5481 {
5482         struct thread *t = &p->threads[p->thread_id];
5483         struct instruction *ip = t->ip;
5484
5485         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5486
5487         JMP_CMP_MI(t, ip, <);
5488 }
5489
5490 static inline void
5491 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5492 {
5493         struct thread *t = &p->threads[p->thread_id];
5494         struct instruction *ip = t->ip;
5495
5496         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5497
5498         JMP_CMP_HI(t, ip, <);
5499 }
5500
5501 static inline void
5502 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5503 {
5504         struct thread *t = &p->threads[p->thread_id];
5505         struct instruction *ip = t->ip;
5506
5507         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5508
5509         JMP_CMP(t, ip, >);
5510 }
5511
5512 static inline void
5513 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5514 {
5515         struct thread *t = &p->threads[p->thread_id];
5516         struct instruction *ip = t->ip;
5517
5518         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5519
5520         JMP_CMP_MH(t, ip, >);
5521 }
5522
5523 static inline void
5524 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5525 {
5526         struct thread *t = &p->threads[p->thread_id];
5527         struct instruction *ip = t->ip;
5528
5529         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5530
5531         JMP_CMP_HM(t, ip, >);
5532 }
5533
5534 static inline void
5535 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5536 {
5537         struct thread *t = &p->threads[p->thread_id];
5538         struct instruction *ip = t->ip;
5539
5540         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5541
5542         JMP_CMP_HH(t, ip, >);
5543 }
5544
5545 static inline void
5546 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5547 {
5548         struct thread *t = &p->threads[p->thread_id];
5549         struct instruction *ip = t->ip;
5550
5551         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5552
5553         JMP_CMP_MI(t, ip, >);
5554 }
5555
5556 static inline void
5557 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5558 {
5559         struct thread *t = &p->threads[p->thread_id];
5560         struct instruction *ip = t->ip;
5561
5562         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5563
5564         JMP_CMP_HI(t, ip, >);
5565 }
5566
5567 /*
5568  * return.
5569  */
5570 static int
5571 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5572                        struct action *action,
5573                        char **tokens __rte_unused,
5574                        int n_tokens,
5575                        struct instruction *instr,
5576                        struct instruction_data *data __rte_unused)
5577 {
5578         CHECK(action, EINVAL);
5579         CHECK(n_tokens == 1, EINVAL);
5580
5581         instr->type = INSTR_RETURN;
5582         return 0;
5583 }
5584
5585 static inline void
5586 instr_return_exec(struct rte_swx_pipeline *p)
5587 {
5588         struct thread *t = &p->threads[p->thread_id];
5589
5590         TRACE("[Thread %2u] return\n", p->thread_id);
5591
5592         t->ip = t->ret;
5593 }
5594
5595 static int
5596 instr_translate(struct rte_swx_pipeline *p,
5597                 struct action *action,
5598                 char *string,
5599                 struct instruction *instr,
5600                 struct instruction_data *data)
5601 {
5602         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5603         int n_tokens = 0, tpos = 0;
5604
5605         /* Parse the instruction string into tokens. */
5606         for ( ; ; ) {
5607                 char *token;
5608
5609                 token = strtok_r(string, " \t\v", &string);
5610                 if (!token)
5611                         break;
5612
5613                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5614                 CHECK_NAME(token, EINVAL);
5615
5616                 tokens[n_tokens] = token;
5617                 n_tokens++;
5618         }
5619
5620         CHECK(n_tokens, EINVAL);
5621
5622         /* Handle the optional instruction label. */
5623         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5624                 strcpy(data->label, tokens[0]);
5625
5626                 tpos += 2;
5627                 CHECK(n_tokens - tpos, EINVAL);
5628         }
5629
5630         /* Identify the instruction type. */
5631         if (!strcmp(tokens[tpos], "rx"))
5632                 return instr_rx_translate(p,
5633                                           action,
5634                                           &tokens[tpos],
5635                                           n_tokens - tpos,
5636                                           instr,
5637                                           data);
5638
5639         if (!strcmp(tokens[tpos], "tx"))
5640                 return instr_tx_translate(p,
5641                                           action,
5642                                           &tokens[tpos],
5643                                           n_tokens - tpos,
5644                                           instr,
5645                                           data);
5646
5647         if (!strcmp(tokens[tpos], "drop"))
5648                 return instr_drop_translate(p,
5649                                             action,
5650                                             &tokens[tpos],
5651                                             n_tokens - tpos,
5652                                             instr,
5653                                             data);
5654
5655         if (!strcmp(tokens[tpos], "extract"))
5656                 return instr_hdr_extract_translate(p,
5657                                                    action,
5658                                                    &tokens[tpos],
5659                                                    n_tokens - tpos,
5660                                                    instr,
5661                                                    data);
5662
5663         if (!strcmp(tokens[tpos], "lookahead"))
5664                 return instr_hdr_lookahead_translate(p,
5665                                                      action,
5666                                                      &tokens[tpos],
5667                                                      n_tokens - tpos,
5668                                                      instr,
5669                                                      data);
5670
5671         if (!strcmp(tokens[tpos], "emit"))
5672                 return instr_hdr_emit_translate(p,
5673                                                 action,
5674                                                 &tokens[tpos],
5675                                                 n_tokens - tpos,
5676                                                 instr,
5677                                                 data);
5678
5679         if (!strcmp(tokens[tpos], "validate"))
5680                 return instr_hdr_validate_translate(p,
5681                                                     action,
5682                                                     &tokens[tpos],
5683                                                     n_tokens - tpos,
5684                                                     instr,
5685                                                     data);
5686
5687         if (!strcmp(tokens[tpos], "invalidate"))
5688                 return instr_hdr_invalidate_translate(p,
5689                                                       action,
5690                                                       &tokens[tpos],
5691                                                       n_tokens - tpos,
5692                                                       instr,
5693                                                       data);
5694
5695         if (!strcmp(tokens[tpos], "mov"))
5696                 return instr_mov_translate(p,
5697                                            action,
5698                                            &tokens[tpos],
5699                                            n_tokens - tpos,
5700                                            instr,
5701                                            data);
5702
5703         if (!strcmp(tokens[tpos], "add"))
5704                 return instr_alu_add_translate(p,
5705                                                action,
5706                                                &tokens[tpos],
5707                                                n_tokens - tpos,
5708                                                instr,
5709                                                data);
5710
5711         if (!strcmp(tokens[tpos], "sub"))
5712                 return instr_alu_sub_translate(p,
5713                                                action,
5714                                                &tokens[tpos],
5715                                                n_tokens - tpos,
5716                                                instr,
5717                                                data);
5718
5719         if (!strcmp(tokens[tpos], "ckadd"))
5720                 return instr_alu_ckadd_translate(p,
5721                                                  action,
5722                                                  &tokens[tpos],
5723                                                  n_tokens - tpos,
5724                                                  instr,
5725                                                  data);
5726
5727         if (!strcmp(tokens[tpos], "cksub"))
5728                 return instr_alu_cksub_translate(p,
5729                                                  action,
5730                                                  &tokens[tpos],
5731                                                  n_tokens - tpos,
5732                                                  instr,
5733                                                  data);
5734
5735         if (!strcmp(tokens[tpos], "and"))
5736                 return instr_alu_and_translate(p,
5737                                                action,
5738                                                &tokens[tpos],
5739                                                n_tokens - tpos,
5740                                                instr,
5741                                                data);
5742
5743         if (!strcmp(tokens[tpos], "or"))
5744                 return instr_alu_or_translate(p,
5745                                               action,
5746                                               &tokens[tpos],
5747                                               n_tokens - tpos,
5748                                               instr,
5749                                               data);
5750
5751         if (!strcmp(tokens[tpos], "xor"))
5752                 return instr_alu_xor_translate(p,
5753                                                action,
5754                                                &tokens[tpos],
5755                                                n_tokens - tpos,
5756                                                instr,
5757                                                data);
5758
5759         if (!strcmp(tokens[tpos], "shl"))
5760                 return instr_alu_shl_translate(p,
5761                                                action,
5762                                                &tokens[tpos],
5763                                                n_tokens - tpos,
5764                                                instr,
5765                                                data);
5766
5767         if (!strcmp(tokens[tpos], "shr"))
5768                 return instr_alu_shr_translate(p,
5769                                                action,
5770                                                &tokens[tpos],
5771                                                n_tokens - tpos,
5772                                                instr,
5773                                                data);
5774
5775         if (!strcmp(tokens[tpos], "regprefetch"))
5776                 return instr_regprefetch_translate(p,
5777                                                    action,
5778                                                    &tokens[tpos],
5779                                                    n_tokens - tpos,
5780                                                    instr,
5781                                                    data);
5782
5783         if (!strcmp(tokens[tpos], "regrd"))
5784                 return instr_regrd_translate(p,
5785                                              action,
5786                                              &tokens[tpos],
5787                                              n_tokens - tpos,
5788                                              instr,
5789                                              data);
5790
5791         if (!strcmp(tokens[tpos], "regwr"))
5792                 return instr_regwr_translate(p,
5793                                              action,
5794                                              &tokens[tpos],
5795                                              n_tokens - tpos,
5796                                              instr,
5797                                              data);
5798
5799         if (!strcmp(tokens[tpos], "regadd"))
5800                 return instr_regadd_translate(p,
5801                                               action,
5802                                               &tokens[tpos],
5803                                               n_tokens - tpos,
5804                                               instr,
5805                                               data);
5806
5807         if (!strcmp(tokens[tpos], "metprefetch"))
5808                 return instr_metprefetch_translate(p,
5809                                                    action,
5810                                                    &tokens[tpos],
5811                                                    n_tokens - tpos,
5812                                                    instr,
5813                                                    data);
5814
5815         if (!strcmp(tokens[tpos], "meter"))
5816                 return instr_meter_translate(p,
5817                                              action,
5818                                              &tokens[tpos],
5819                                              n_tokens - tpos,
5820                                              instr,
5821                                              data);
5822
5823         if (!strcmp(tokens[tpos], "table"))
5824                 return instr_table_translate(p,
5825                                              action,
5826                                              &tokens[tpos],
5827                                              n_tokens - tpos,
5828                                              instr,
5829                                              data);
5830
5831         if (!strcmp(tokens[tpos], "learn"))
5832                 return instr_learn_translate(p,
5833                                              action,
5834                                              &tokens[tpos],
5835                                              n_tokens - tpos,
5836                                              instr,
5837                                              data);
5838
5839         if (!strcmp(tokens[tpos], "forget"))
5840                 return instr_forget_translate(p,
5841                                               action,
5842                                               &tokens[tpos],
5843                                               n_tokens - tpos,
5844                                               instr,
5845                                               data);
5846
5847         if (!strcmp(tokens[tpos], "extern"))
5848                 return instr_extern_translate(p,
5849                                               action,
5850                                               &tokens[tpos],
5851                                               n_tokens - tpos,
5852                                               instr,
5853                                               data);
5854
5855         if (!strcmp(tokens[tpos], "jmp"))
5856                 return instr_jmp_translate(p,
5857                                            action,
5858                                            &tokens[tpos],
5859                                            n_tokens - tpos,
5860                                            instr,
5861                                            data);
5862
5863         if (!strcmp(tokens[tpos], "jmpv"))
5864                 return instr_jmp_valid_translate(p,
5865                                                  action,
5866                                                  &tokens[tpos],
5867                                                  n_tokens - tpos,
5868                                                  instr,
5869                                                  data);
5870
5871         if (!strcmp(tokens[tpos], "jmpnv"))
5872                 return instr_jmp_invalid_translate(p,
5873                                                    action,
5874                                                    &tokens[tpos],
5875                                                    n_tokens - tpos,
5876                                                    instr,
5877                                                    data);
5878
5879         if (!strcmp(tokens[tpos], "jmph"))
5880                 return instr_jmp_hit_translate(p,
5881                                                action,
5882                                                &tokens[tpos],
5883                                                n_tokens - tpos,
5884                                                instr,
5885                                                data);
5886
5887         if (!strcmp(tokens[tpos], "jmpnh"))
5888                 return instr_jmp_miss_translate(p,
5889                                                 action,
5890                                                 &tokens[tpos],
5891                                                 n_tokens - tpos,
5892                                                 instr,
5893                                                 data);
5894
5895         if (!strcmp(tokens[tpos], "jmpa"))
5896                 return instr_jmp_action_hit_translate(p,
5897                                                       action,
5898                                                       &tokens[tpos],
5899                                                       n_tokens - tpos,
5900                                                       instr,
5901                                                       data);
5902
5903         if (!strcmp(tokens[tpos], "jmpna"))
5904                 return instr_jmp_action_miss_translate(p,
5905                                                        action,
5906                                                        &tokens[tpos],
5907                                                        n_tokens - tpos,
5908                                                        instr,
5909                                                        data);
5910
5911         if (!strcmp(tokens[tpos], "jmpeq"))
5912                 return instr_jmp_eq_translate(p,
5913                                               action,
5914                                               &tokens[tpos],
5915                                               n_tokens - tpos,
5916                                               instr,
5917                                               data);
5918
5919         if (!strcmp(tokens[tpos], "jmpneq"))
5920                 return instr_jmp_neq_translate(p,
5921                                                action,
5922                                                &tokens[tpos],
5923                                                n_tokens - tpos,
5924                                                instr,
5925                                                data);
5926
5927         if (!strcmp(tokens[tpos], "jmplt"))
5928                 return instr_jmp_lt_translate(p,
5929                                               action,
5930                                               &tokens[tpos],
5931                                               n_tokens - tpos,
5932                                               instr,
5933                                               data);
5934
5935         if (!strcmp(tokens[tpos], "jmpgt"))
5936                 return instr_jmp_gt_translate(p,
5937                                               action,
5938                                               &tokens[tpos],
5939                                               n_tokens - tpos,
5940                                               instr,
5941                                               data);
5942
5943         if (!strcmp(tokens[tpos], "return"))
5944                 return instr_return_translate(p,
5945                                               action,
5946                                               &tokens[tpos],
5947                                               n_tokens - tpos,
5948                                               instr,
5949                                               data);
5950
5951         return -EINVAL;
5952 }
5953
5954 static struct instruction_data *
5955 label_find(struct instruction_data *data, uint32_t n, const char *label)
5956 {
5957         uint32_t i;
5958
5959         for (i = 0; i < n; i++)
5960                 if (!strcmp(label, data[i].label))
5961                         return &data[i];
5962
5963         return NULL;
5964 }
5965
5966 static uint32_t
5967 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5968 {
5969         uint32_t count = 0, i;
5970
5971         if (!label[0])
5972                 return 0;
5973
5974         for (i = 0; i < n; i++)
5975                 if (!strcmp(label, data[i].jmp_label))
5976                         count++;
5977
5978         return count;
5979 }
5980
5981 static int
5982 instr_label_check(struct instruction_data *instruction_data,
5983                   uint32_t n_instructions)
5984 {
5985         uint32_t i;
5986
5987         /* Check that all instruction labels are unique. */
5988         for (i = 0; i < n_instructions; i++) {
5989                 struct instruction_data *data = &instruction_data[i];
5990                 char *label = data->label;
5991                 uint32_t j;
5992
5993                 if (!label[0])
5994                         continue;
5995
5996                 for (j = i + 1; j < n_instructions; j++)
5997                         CHECK(strcmp(label, instruction_data[j].label), EINVAL);
5998         }
5999
6000         /* Get users for each instruction label. */
6001         for (i = 0; i < n_instructions; i++) {
6002                 struct instruction_data *data = &instruction_data[i];
6003                 char *label = data->label;
6004
6005                 data->n_users = label_is_used(instruction_data,
6006                                               n_instructions,
6007                                               label);
6008         }
6009
6010         return 0;
6011 }
6012
6013 static int
6014 instr_jmp_resolve(struct instruction *instructions,
6015                   struct instruction_data *instruction_data,
6016                   uint32_t n_instructions)
6017 {
6018         uint32_t i;
6019
6020         for (i = 0; i < n_instructions; i++) {
6021                 struct instruction *instr = &instructions[i];
6022                 struct instruction_data *data = &instruction_data[i];
6023                 struct instruction_data *found;
6024
6025                 if (!instruction_is_jmp(instr))
6026                         continue;
6027
6028                 found = label_find(instruction_data,
6029                                    n_instructions,
6030                                    data->jmp_label);
6031                 CHECK(found, EINVAL);
6032
6033                 instr->jmp.ip = &instructions[found - instruction_data];
6034         }
6035
6036         return 0;
6037 }
6038
6039 static int
6040 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6041              struct action *a,
6042              struct instruction *instr,
6043              struct instruction_data *data __rte_unused,
6044              uint32_t n_instructions)
6045 {
6046         if (!a) {
6047                 enum instruction_type type;
6048                 uint32_t i;
6049
6050                 /* Check that the first instruction is rx. */
6051                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6052
6053                 /* Check that there is at least one tx instruction. */
6054                 for (i = 0; i < n_instructions; i++) {
6055                         type = instr[i].type;
6056
6057                         if (instruction_is_tx(type))
6058                                 break;
6059                 }
6060                 CHECK(i < n_instructions, EINVAL);
6061
6062                 /* Check that the last instruction is either tx or unconditional
6063                  * jump.
6064                  */
6065                 type = instr[n_instructions - 1].type;
6066                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6067         }
6068
6069         if (a) {
6070                 enum instruction_type type;
6071                 uint32_t i;
6072
6073                 /* Check that there is at least one return or tx instruction. */
6074                 for (i = 0; i < n_instructions; i++) {
6075                         type = instr[i].type;
6076
6077                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6078                                 break;
6079                 }
6080                 CHECK(i < n_instructions, EINVAL);
6081         }
6082
6083         return 0;
6084 }
6085
6086 static uint32_t
6087 instr_compact(struct instruction *instructions,
6088               struct instruction_data *instruction_data,
6089               uint32_t n_instructions)
6090 {
6091         uint32_t i, pos = 0;
6092
6093         /* Eliminate the invalid instructions that have been optimized out. */
6094         for (i = 0; i < n_instructions; i++) {
6095                 struct instruction *instr = &instructions[i];
6096                 struct instruction_data *data = &instruction_data[i];
6097
6098                 if (data->invalid)
6099                         continue;
6100
6101                 if (i != pos) {
6102                         memcpy(&instructions[pos], instr, sizeof(*instr));
6103                         memcpy(&instruction_data[pos], data, sizeof(*data));
6104                 }
6105
6106                 pos++;
6107         }
6108
6109         return pos;
6110 }
6111
6112 static int
6113 instr_pattern_extract_many_search(struct instruction *instr,
6114                                   struct instruction_data *data,
6115                                   uint32_t n_instr,
6116                                   uint32_t *n_pattern_instr)
6117 {
6118         uint32_t i;
6119
6120         for (i = 0; i < n_instr; i++) {
6121                 if (data[i].invalid)
6122                         break;
6123
6124                 if (instr[i].type != INSTR_HDR_EXTRACT)
6125                         break;
6126
6127                 if (i == RTE_DIM(instr->io.hdr.header_id))
6128                         break;
6129
6130                 if (i && data[i].n_users)
6131                         break;
6132         }
6133
6134         if (i < 2)
6135                 return 0;
6136
6137         *n_pattern_instr = i;
6138         return 1;
6139 }
6140
6141 static void
6142 instr_pattern_extract_many_replace(struct instruction *instr,
6143                                    struct instruction_data *data,
6144                                    uint32_t n_instr)
6145 {
6146         uint32_t i;
6147
6148         for (i = 1; i < n_instr; i++) {
6149                 instr[0].type++;
6150                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6151                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6152                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6153
6154                 data[i].invalid = 1;
6155         }
6156 }
6157
6158 static uint32_t
6159 instr_pattern_extract_many_optimize(struct instruction *instructions,
6160                                     struct instruction_data *instruction_data,
6161                                     uint32_t n_instructions)
6162 {
6163         uint32_t i;
6164
6165         for (i = 0; i < n_instructions; ) {
6166                 struct instruction *instr = &instructions[i];
6167                 struct instruction_data *data = &instruction_data[i];
6168                 uint32_t n_instr = 0;
6169                 int detected;
6170
6171                 /* Extract many. */
6172                 detected = instr_pattern_extract_many_search(instr,
6173                                                              data,
6174                                                              n_instructions - i,
6175                                                              &n_instr);
6176                 if (detected) {
6177                         instr_pattern_extract_many_replace(instr,
6178                                                            data,
6179                                                            n_instr);
6180                         i += n_instr;
6181                         continue;
6182                 }
6183
6184                 /* No pattern starting at the current instruction. */
6185                 i++;
6186         }
6187
6188         /* Eliminate the invalid instructions that have been optimized out. */
6189         n_instructions = instr_compact(instructions,
6190                                        instruction_data,
6191                                        n_instructions);
6192
6193         return n_instructions;
6194 }
6195
6196 static int
6197 instr_pattern_emit_many_tx_search(struct instruction *instr,
6198                                   struct instruction_data *data,
6199                                   uint32_t n_instr,
6200                                   uint32_t *n_pattern_instr)
6201 {
6202         uint32_t i;
6203
6204         for (i = 0; i < n_instr; i++) {
6205                 if (data[i].invalid)
6206                         break;
6207
6208                 if (instr[i].type != INSTR_HDR_EMIT)
6209                         break;
6210
6211                 if (i == RTE_DIM(instr->io.hdr.header_id))
6212                         break;
6213
6214                 if (i && data[i].n_users)
6215                         break;
6216         }
6217
6218         if (!i)
6219                 return 0;
6220
6221         if (instr[i].type != INSTR_TX)
6222                 return 0;
6223
6224         if (data[i].n_users)
6225                 return 0;
6226
6227         i++;
6228
6229         *n_pattern_instr = i;
6230         return 1;
6231 }
6232
6233 static void
6234 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6235                                    struct instruction_data *data,
6236                                    uint32_t n_instr)
6237 {
6238         uint32_t i;
6239
6240         /* Any emit instruction in addition to the first one. */
6241         for (i = 1; i < n_instr - 1; i++) {
6242                 instr[0].type++;
6243                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6244                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6245                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6246
6247                 data[i].invalid = 1;
6248         }
6249
6250         /* The TX instruction is the last one in the pattern. */
6251         instr[0].type++;
6252         instr[0].io.io.offset = instr[i].io.io.offset;
6253         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6254         data[i].invalid = 1;
6255 }
6256
6257 static uint32_t
6258 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6259                                     struct instruction_data *instruction_data,
6260                                     uint32_t n_instructions)
6261 {
6262         uint32_t i;
6263
6264         for (i = 0; i < n_instructions; ) {
6265                 struct instruction *instr = &instructions[i];
6266                 struct instruction_data *data = &instruction_data[i];
6267                 uint32_t n_instr = 0;
6268                 int detected;
6269
6270                 /* Emit many + TX. */
6271                 detected = instr_pattern_emit_many_tx_search(instr,
6272                                                              data,
6273                                                              n_instructions - i,
6274                                                              &n_instr);
6275                 if (detected) {
6276                         instr_pattern_emit_many_tx_replace(instr,
6277                                                            data,
6278                                                            n_instr);
6279                         i += n_instr;
6280                         continue;
6281                 }
6282
6283                 /* No pattern starting at the current instruction. */
6284                 i++;
6285         }
6286
6287         /* Eliminate the invalid instructions that have been optimized out. */
6288         n_instructions = instr_compact(instructions,
6289                                        instruction_data,
6290                                        n_instructions);
6291
6292         return n_instructions;
6293 }
6294
6295 static uint32_t
6296 action_arg_src_mov_count(struct action *a,
6297                          uint32_t arg_id,
6298                          struct instruction *instructions,
6299                          struct instruction_data *instruction_data,
6300                          uint32_t n_instructions);
6301
6302 static int
6303 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6304                                       struct action *a,
6305                                       struct instruction *instr,
6306                                       struct instruction_data *data,
6307                                       uint32_t n_instr,
6308                                       struct instruction *instructions,
6309                                       struct instruction_data *instruction_data,
6310                                       uint32_t n_instructions,
6311                                       uint32_t *n_pattern_instr)
6312 {
6313         struct header *h;
6314         uint32_t src_field_id, i, j;
6315
6316         /* Prerequisites. */
6317         if (!a || !a->st)
6318                 return 0;
6319
6320         /* First instruction: MOV_HM. */
6321         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6322                 return 0;
6323
6324         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6325         if (!h || h->st->var_size)
6326                 return 0;
6327
6328         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6329                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6330                         break;
6331
6332         if (src_field_id == a->st->n_fields)
6333                 return 0;
6334
6335         if (instr[0].mov.dst.offset ||
6336             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6337             instr[0].mov.src.struct_id ||
6338             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6339             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6340                 return 0;
6341
6342         if ((n_instr < h->st->n_fields + 1) ||
6343              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6344                 return 0;
6345
6346         /* Subsequent instructions: MOV_HM. */
6347         for (i = 1; i < h->st->n_fields; i++)
6348                 if (data[i].invalid ||
6349                     data[i].n_users ||
6350                     (instr[i].type != INSTR_MOV_HM) ||
6351                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6352                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6353                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6354                     instr[i].mov.src.struct_id ||
6355                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6356                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6357                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6358                         return 0;
6359
6360         /* Last instruction: HDR_VALIDATE. */
6361         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6362             (instr[i].valid.header_id != h->id))
6363                 return 0;
6364
6365         /* Check that none of the action args that are used as source for this
6366          * DMA transfer are not used as source in any other mov instruction.
6367          */
6368         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6369                 uint32_t n_users;
6370
6371                 n_users = action_arg_src_mov_count(a,
6372                                                    j,
6373                                                    instructions,
6374                                                    instruction_data,
6375                                                    n_instructions);
6376                 if (n_users > 1)
6377                         return 0;
6378         }
6379
6380         *n_pattern_instr = 1 + i;
6381         return 1;
6382 }
6383
6384 static void
6385 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6386                                        struct action *a,
6387                                        struct instruction *instr,
6388                                        struct instruction_data *data,
6389                                        uint32_t n_instr)
6390 {
6391         struct header *h;
6392         uint32_t src_field_id, src_offset, i;
6393
6394         /* Read from the instructions before they are modified. */
6395         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6396         if (!h)
6397                 return;
6398
6399         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6400                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6401                         break;
6402
6403         if (src_field_id == a->st->n_fields)
6404                 return;
6405
6406         src_offset = instr[0].mov.src.offset;
6407
6408         /* Modify the instructions. */
6409         instr[0].type = INSTR_DMA_HT;
6410         instr[0].dma.dst.header_id[0] = h->id;
6411         instr[0].dma.dst.struct_id[0] = h->struct_id;
6412         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6413         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6414
6415         for (i = 1; i < n_instr; i++)
6416                 data[i].invalid = 1;
6417
6418         /* Update the endianness of the action arguments to header endianness. */
6419         for (i = 0; i < h->st->n_fields; i++)
6420                 a->args_endianness[src_field_id + i] = 1;
6421 }
6422
6423 static uint32_t
6424 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6425                                         struct action *a,
6426                                         struct instruction *instructions,
6427                                         struct instruction_data *instruction_data,
6428                                         uint32_t n_instructions)
6429 {
6430         uint32_t i;
6431
6432         if (!a || !a->st)
6433                 return n_instructions;
6434
6435         for (i = 0; i < n_instructions; ) {
6436                 struct instruction *instr = &instructions[i];
6437                 struct instruction_data *data = &instruction_data[i];
6438                 uint32_t n_instr = 0;
6439                 int detected;
6440
6441                 /* Mov all + validate. */
6442                 detected = instr_pattern_mov_all_validate_search(p,
6443                                                                  a,
6444                                                                  instr,
6445                                                                  data,
6446                                                                  n_instructions - i,
6447                                                                  instructions,
6448                                                                  instruction_data,
6449                                                                  n_instructions,
6450                                                                  &n_instr);
6451                 if (detected) {
6452                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6453                         i += n_instr;
6454                         continue;
6455                 }
6456
6457                 /* No pattern starting at the current instruction. */
6458                 i++;
6459         }
6460
6461         /* Eliminate the invalid instructions that have been optimized out. */
6462         n_instructions = instr_compact(instructions,
6463                                        instruction_data,
6464                                        n_instructions);
6465
6466         return n_instructions;
6467 }
6468
6469 static int
6470 instr_pattern_dma_many_search(struct instruction *instr,
6471                               struct instruction_data *data,
6472                               uint32_t n_instr,
6473                               uint32_t *n_pattern_instr)
6474 {
6475         uint32_t i;
6476
6477         for (i = 0; i < n_instr; i++) {
6478                 if (data[i].invalid)
6479                         break;
6480
6481                 if (instr[i].type != INSTR_DMA_HT)
6482                         break;
6483
6484                 if (i == RTE_DIM(instr->dma.dst.header_id))
6485                         break;
6486
6487                 if (i && data[i].n_users)
6488                         break;
6489         }
6490
6491         if (i < 2)
6492                 return 0;
6493
6494         *n_pattern_instr = i;
6495         return 1;
6496 }
6497
6498 static void
6499 instr_pattern_dma_many_replace(struct instruction *instr,
6500                                struct instruction_data *data,
6501                                uint32_t n_instr)
6502 {
6503         uint32_t i;
6504
6505         for (i = 1; i < n_instr; i++) {
6506                 instr[0].type++;
6507                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6508                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6509                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6510                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6511
6512                 data[i].invalid = 1;
6513         }
6514 }
6515
6516 static uint32_t
6517 instr_pattern_dma_many_optimize(struct instruction *instructions,
6518                struct instruction_data *instruction_data,
6519                uint32_t n_instructions)
6520 {
6521         uint32_t i;
6522
6523         for (i = 0; i < n_instructions; ) {
6524                 struct instruction *instr = &instructions[i];
6525                 struct instruction_data *data = &instruction_data[i];
6526                 uint32_t n_instr = 0;
6527                 int detected;
6528
6529                 /* DMA many. */
6530                 detected = instr_pattern_dma_many_search(instr,
6531                                                          data,
6532                                                          n_instructions - i,
6533                                                          &n_instr);
6534                 if (detected) {
6535                         instr_pattern_dma_many_replace(instr, data, n_instr);
6536                         i += n_instr;
6537                         continue;
6538                 }
6539
6540                 /* No pattern starting at the current instruction. */
6541                 i++;
6542         }
6543
6544         /* Eliminate the invalid instructions that have been optimized out. */
6545         n_instructions = instr_compact(instructions,
6546                                        instruction_data,
6547                                        n_instructions);
6548
6549         return n_instructions;
6550 }
6551
6552 static uint32_t
6553 instr_optimize(struct rte_swx_pipeline *p,
6554                struct action *a,
6555                struct instruction *instructions,
6556                struct instruction_data *instruction_data,
6557                uint32_t n_instructions)
6558 {
6559         /* Extract many. */
6560         n_instructions = instr_pattern_extract_many_optimize(instructions,
6561                                                              instruction_data,
6562                                                              n_instructions);
6563
6564         /* Emit many + TX. */
6565         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6566                                                              instruction_data,
6567                                                              n_instructions);
6568
6569         /* Mov all + validate. */
6570         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6571                                                                  a,
6572                                                                  instructions,
6573                                                                  instruction_data,
6574                                                                  n_instructions);
6575
6576         /* DMA many. */
6577         n_instructions = instr_pattern_dma_many_optimize(instructions,
6578                                                          instruction_data,
6579                                                          n_instructions);
6580
6581         return n_instructions;
6582 }
6583
6584 static int
6585 instruction_config(struct rte_swx_pipeline *p,
6586                    struct action *a,
6587                    const char **instructions,
6588                    uint32_t n_instructions)
6589 {
6590         struct instruction *instr = NULL;
6591         struct instruction_data *data = NULL;
6592         int err = 0;
6593         uint32_t i;
6594
6595         CHECK(n_instructions, EINVAL);
6596         CHECK(instructions, EINVAL);
6597         for (i = 0; i < n_instructions; i++)
6598                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6599
6600         /* Memory allocation. */
6601         instr = calloc(n_instructions, sizeof(struct instruction));
6602         if (!instr) {
6603                 err = -ENOMEM;
6604                 goto error;
6605         }
6606
6607         data = calloc(n_instructions, sizeof(struct instruction_data));
6608         if (!data) {
6609                 err = -ENOMEM;
6610                 goto error;
6611         }
6612
6613         for (i = 0; i < n_instructions; i++) {
6614                 char *string = strdup(instructions[i]);
6615                 if (!string) {
6616                         err = -ENOMEM;
6617                         goto error;
6618                 }
6619
6620                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6621                 if (err) {
6622                         free(string);
6623                         goto error;
6624                 }
6625
6626                 free(string);
6627         }
6628
6629         err = instr_label_check(data, n_instructions);
6630         if (err)
6631                 goto error;
6632
6633         err = instr_verify(p, a, instr, data, n_instructions);
6634         if (err)
6635                 goto error;
6636
6637         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6638
6639         err = instr_jmp_resolve(instr, data, n_instructions);
6640         if (err)
6641                 goto error;
6642
6643         if (a) {
6644                 a->instructions = instr;
6645                 a->instruction_data = data;
6646                 a->n_instructions = n_instructions;
6647         } else {
6648                 p->instructions = instr;
6649                 p->instruction_data = data;
6650                 p->n_instructions = n_instructions;
6651         }
6652
6653         return 0;
6654
6655 error:
6656         free(data);
6657         free(instr);
6658         return err;
6659 }
6660
6661 static instr_exec_t instruction_table[] = {
6662         [INSTR_RX] = instr_rx_exec,
6663         [INSTR_TX] = instr_tx_exec,
6664         [INSTR_TX_I] = instr_tx_i_exec,
6665         [INSTR_DROP] = instr_drop_exec,
6666
6667         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6668         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6669         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6670         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6671         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6672         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6673         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6674         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6675         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6676         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6677
6678         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6679         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6680         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6681         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6682         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6683         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6684         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6685         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6686         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6687
6688         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6689         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6690
6691         [INSTR_MOV] = instr_mov_exec,
6692         [INSTR_MOV_MH] = instr_mov_mh_exec,
6693         [INSTR_MOV_HM] = instr_mov_hm_exec,
6694         [INSTR_MOV_HH] = instr_mov_hh_exec,
6695         [INSTR_MOV_I] = instr_mov_i_exec,
6696
6697         [INSTR_DMA_HT] = instr_dma_ht_exec,
6698         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6699         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6700         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6701         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6702         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6703         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6704         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6705
6706         [INSTR_ALU_ADD] = instr_alu_add_exec,
6707         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6708         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6709         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6710         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6711         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6712
6713         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6714         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6715         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6716         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6717         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6718         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6719
6720         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6721         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6722         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6723         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6724
6725         [INSTR_ALU_AND] = instr_alu_and_exec,
6726         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6727         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6728         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6729         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6730
6731         [INSTR_ALU_OR] = instr_alu_or_exec,
6732         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6733         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6734         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6735         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6736
6737         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6738         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6739         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6740         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6741         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6742
6743         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6744         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6745         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6746         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6747         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6748         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6749
6750         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6751         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6752         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6753         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6754         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6755         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6756
6757         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6758         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6759         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6760
6761         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6762         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6763         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6764         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6765         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6766         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6767
6768         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6769         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6770         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6771         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6772         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6773         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6774         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6775         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6776         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6777
6778         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6779         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6780         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6781         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6782         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6783         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6784         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6785         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6786         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6787
6788         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6789         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6790         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6791
6792         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6793         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6794         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6795         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6796         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6797         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6798         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6799         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6800         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6801         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6802         [INSTR_METER_IMM] = instr_meter_imm_exec,
6803         [INSTR_METER_IMI] = instr_meter_imi_exec,
6804
6805         [INSTR_TABLE] = instr_table_exec,
6806         [INSTR_TABLE_AF] = instr_table_af_exec,
6807         [INSTR_SELECTOR] = instr_selector_exec,
6808         [INSTR_LEARNER] = instr_learner_exec,
6809         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6810         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6811         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6812         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6813         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6814
6815         [INSTR_JMP] = instr_jmp_exec,
6816         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6817         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6818         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6819         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6820         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6821         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6822
6823         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6824         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6825         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6826         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6827         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6828
6829         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6830         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6831         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6832         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6833         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6834
6835         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6836         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6837         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6838         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6839         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6840         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6841
6842         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6843         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6844         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6845         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6846         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6847         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6848
6849         [INSTR_RETURN] = instr_return_exec,
6850 };
6851
6852 static int
6853 instruction_table_build(struct rte_swx_pipeline *p)
6854 {
6855         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6856                                       sizeof(struct instr_exec_t *));
6857         if (!p->instruction_table)
6858                 return -EINVAL;
6859
6860         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6861
6862         return 0;
6863 }
6864
6865 static void
6866 instruction_table_build_free(struct rte_swx_pipeline *p)
6867 {
6868         if (!p->instruction_table)
6869                 return;
6870
6871         free(p->instruction_table);
6872         p->instruction_table = NULL;
6873 }
6874
6875 static void
6876 instruction_table_free(struct rte_swx_pipeline *p)
6877 {
6878         instruction_table_build_free(p);
6879 }
6880
6881 static inline void
6882 instr_exec(struct rte_swx_pipeline *p)
6883 {
6884         struct thread *t = &p->threads[p->thread_id];
6885         struct instruction *ip = t->ip;
6886         instr_exec_t instr = p->instruction_table[ip->type];
6887
6888         instr(p);
6889 }
6890
6891 /*
6892  * Action.
6893  */
6894 static struct action *
6895 action_find(struct rte_swx_pipeline *p, const char *name)
6896 {
6897         struct action *elem;
6898
6899         if (!name)
6900                 return NULL;
6901
6902         TAILQ_FOREACH(elem, &p->actions, node)
6903                 if (strcmp(elem->name, name) == 0)
6904                         return elem;
6905
6906         return NULL;
6907 }
6908
6909 static struct action *
6910 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6911 {
6912         struct action *action = NULL;
6913
6914         TAILQ_FOREACH(action, &p->actions, node)
6915                 if (action->id == id)
6916                         return action;
6917
6918         return NULL;
6919 }
6920
6921 static struct field *
6922 action_field_find(struct action *a, const char *name)
6923 {
6924         return a->st ? struct_type_field_find(a->st, name) : NULL;
6925 }
6926
6927 static struct field *
6928 action_field_parse(struct action *action, const char *name)
6929 {
6930         if (name[0] != 't' || name[1] != '.')
6931                 return NULL;
6932
6933         return action_field_find(action, &name[2]);
6934 }
6935
6936 static int
6937 action_has_nbo_args(struct action *a)
6938 {
6939         uint32_t i;
6940
6941         /* Return if the action does not have any args. */
6942         if (!a->st)
6943                 return 0; /* FALSE */
6944
6945         for (i = 0; i < a->st->n_fields; i++)
6946                 if (a->args_endianness[i])
6947                         return 1; /* TRUE */
6948
6949         return 0; /* FALSE */
6950 }
6951
6952 static int
6953 action_does_learning(struct action *a)
6954 {
6955         uint32_t i;
6956
6957         for (i = 0; i < a->n_instructions; i++)
6958                 switch (a->instructions[i].type) {
6959                 case INSTR_LEARNER_LEARN:
6960                         return 1; /* TRUE */
6961
6962                 case INSTR_LEARNER_FORGET:
6963                         return 1; /* TRUE */
6964
6965                 default:
6966                         continue;
6967                 }
6968
6969         return 0; /* FALSE */
6970 }
6971
6972 int
6973 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6974                                const char *name,
6975                                const char *args_struct_type_name,
6976                                const char **instructions,
6977                                uint32_t n_instructions)
6978 {
6979         struct struct_type *args_struct_type = NULL;
6980         struct action *a;
6981         int err;
6982
6983         CHECK(p, EINVAL);
6984
6985         CHECK_NAME(name, EINVAL);
6986         CHECK(!action_find(p, name), EEXIST);
6987
6988         if (args_struct_type_name) {
6989                 CHECK_NAME(args_struct_type_name, EINVAL);
6990                 args_struct_type = struct_type_find(p, args_struct_type_name);
6991                 CHECK(args_struct_type, EINVAL);
6992                 CHECK(!args_struct_type->var_size, EINVAL);
6993         }
6994
6995         /* Node allocation. */
6996         a = calloc(1, sizeof(struct action));
6997         CHECK(a, ENOMEM);
6998         if (args_struct_type) {
6999                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7000                 if (!a->args_endianness) {
7001                         free(a);
7002                         CHECK(0, ENOMEM);
7003                 }
7004         }
7005
7006         /* Node initialization. */
7007         strcpy(a->name, name);
7008         a->st = args_struct_type;
7009         a->id = p->n_actions;
7010
7011         /* Instruction translation. */
7012         err = instruction_config(p, a, instructions, n_instructions);
7013         if (err) {
7014                 free(a->args_endianness);
7015                 free(a);
7016                 return err;
7017         }
7018
7019         /* Node add to tailq. */
7020         TAILQ_INSERT_TAIL(&p->actions, a, node);
7021         p->n_actions++;
7022
7023         return 0;
7024 }
7025
7026 static int
7027 action_build(struct rte_swx_pipeline *p)
7028 {
7029         struct action *action;
7030
7031         /* p->action_instructions. */
7032         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7033         CHECK(p->action_instructions, ENOMEM);
7034
7035         TAILQ_FOREACH(action, &p->actions, node)
7036                 p->action_instructions[action->id] = action->instructions;
7037
7038         /* p->action_funcs. */
7039         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7040         CHECK(p->action_funcs, ENOMEM);
7041
7042         return 0;
7043 }
7044
7045 static void
7046 action_build_free(struct rte_swx_pipeline *p)
7047 {
7048         free(p->action_funcs);
7049         p->action_funcs = NULL;
7050
7051         free(p->action_instructions);
7052         p->action_instructions = NULL;
7053 }
7054
7055 static void
7056 action_free(struct rte_swx_pipeline *p)
7057 {
7058         action_build_free(p);
7059
7060         for ( ; ; ) {
7061                 struct action *action;
7062
7063                 action = TAILQ_FIRST(&p->actions);
7064                 if (!action)
7065                         break;
7066
7067                 TAILQ_REMOVE(&p->actions, action, node);
7068                 free(action->instruction_data);
7069                 free(action->instructions);
7070                 free(action);
7071         }
7072 }
7073
7074 static uint32_t
7075 action_arg_src_mov_count(struct action *a,
7076                          uint32_t arg_id,
7077                          struct instruction *instructions,
7078                          struct instruction_data *instruction_data,
7079                          uint32_t n_instructions)
7080 {
7081         uint32_t offset, n_users = 0, i;
7082
7083         if (!a->st ||
7084             (arg_id >= a->st->n_fields) ||
7085             !instructions ||
7086             !instruction_data ||
7087             !n_instructions)
7088                 return 0;
7089
7090         offset = a->st->fields[arg_id].offset / 8;
7091
7092         for (i = 0; i < n_instructions; i++) {
7093                 struct instruction *instr = &instructions[i];
7094                 struct instruction_data *data = &instruction_data[i];
7095
7096                 if (data->invalid ||
7097                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7098                     instr->mov.src.struct_id ||
7099                     (instr->mov.src.offset != offset))
7100                         continue;
7101
7102                 n_users++;
7103         }
7104
7105         return n_users;
7106 }
7107
7108 /*
7109  * Table.
7110  */
7111 static struct table_type *
7112 table_type_find(struct rte_swx_pipeline *p, const char *name)
7113 {
7114         struct table_type *elem;
7115
7116         TAILQ_FOREACH(elem, &p->table_types, node)
7117                 if (strcmp(elem->name, name) == 0)
7118                         return elem;
7119
7120         return NULL;
7121 }
7122
7123 static struct table_type *
7124 table_type_resolve(struct rte_swx_pipeline *p,
7125                    const char *recommended_type_name,
7126                    enum rte_swx_table_match_type match_type)
7127 {
7128         struct table_type *elem;
7129
7130         /* Only consider the recommended type if the match type is correct. */
7131         if (recommended_type_name)
7132                 TAILQ_FOREACH(elem, &p->table_types, node)
7133                         if (!strcmp(elem->name, recommended_type_name) &&
7134                             (elem->match_type == match_type))
7135                                 return elem;
7136
7137         /* Ignore the recommended type and get the first element with this match
7138          * type.
7139          */
7140         TAILQ_FOREACH(elem, &p->table_types, node)
7141                 if (elem->match_type == match_type)
7142                         return elem;
7143
7144         return NULL;
7145 }
7146
7147 static struct table *
7148 table_find(struct rte_swx_pipeline *p, const char *name)
7149 {
7150         struct table *elem;
7151
7152         TAILQ_FOREACH(elem, &p->tables, node)
7153                 if (strcmp(elem->name, name) == 0)
7154                         return elem;
7155
7156         return NULL;
7157 }
7158
7159 static struct table *
7160 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7161 {
7162         struct table *table = NULL;
7163
7164         TAILQ_FOREACH(table, &p->tables, node)
7165                 if (table->id == id)
7166                         return table;
7167
7168         return NULL;
7169 }
7170
7171 int
7172 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7173                                      const char *name,
7174                                      enum rte_swx_table_match_type match_type,
7175                                      struct rte_swx_table_ops *ops)
7176 {
7177         struct table_type *elem;
7178
7179         CHECK(p, EINVAL);
7180
7181         CHECK_NAME(name, EINVAL);
7182         CHECK(!table_type_find(p, name), EEXIST);
7183
7184         CHECK(ops, EINVAL);
7185         CHECK(ops->create, EINVAL);
7186         CHECK(ops->lkp, EINVAL);
7187         CHECK(ops->free, EINVAL);
7188
7189         /* Node allocation. */
7190         elem = calloc(1, sizeof(struct table_type));
7191         CHECK(elem, ENOMEM);
7192
7193         /* Node initialization. */
7194         strcpy(elem->name, name);
7195         elem->match_type = match_type;
7196         memcpy(&elem->ops, ops, sizeof(*ops));
7197
7198         /* Node add to tailq. */
7199         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7200
7201         return 0;
7202 }
7203
7204 static int
7205 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7206                          uint32_t n_fields,
7207                          enum rte_swx_table_match_type *match_type)
7208 {
7209         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7210
7211         for (i = 0; i < n_fields; i++) {
7212                 struct rte_swx_match_field_params  *f = &fields[i];
7213
7214                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7215                         n_fields_em++;
7216
7217                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7218                         n_fields_lpm++;
7219         }
7220
7221         if ((n_fields_lpm > 1) ||
7222             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7223                 return -EINVAL;
7224
7225         *match_type = (n_fields_em == n_fields) ?
7226                        RTE_SWX_TABLE_MATCH_EXACT :
7227                        RTE_SWX_TABLE_MATCH_WILDCARD;
7228
7229         return 0;
7230 }
7231
7232 static int
7233 table_match_fields_check(struct rte_swx_pipeline *p,
7234                          struct rte_swx_pipeline_table_params *params,
7235                          struct header **header)
7236 {
7237         struct header *h0 = NULL;
7238         struct field *hf, *mf;
7239         uint32_t *offset = NULL, i;
7240         int status = 0;
7241
7242         /* Return if no match fields. */
7243         if (!params->n_fields) {
7244                 if (params->fields) {
7245                         status = -EINVAL;
7246                         goto end;
7247                 }
7248
7249                 if (header)
7250                         *header = NULL;
7251
7252                 return 0;
7253         }
7254
7255         /* Memory allocation. */
7256         offset = calloc(params->n_fields, sizeof(uint32_t));
7257         if (!offset) {
7258                 status = -ENOMEM;
7259                 goto end;
7260         }
7261
7262         /* Check that all the match fields belong to either the same header or
7263          * to the meta-data.
7264          */
7265         hf = header_field_parse(p, params->fields[0].name, &h0);
7266         mf = metadata_field_parse(p, params->fields[0].name);
7267         if ((!hf && !mf) || (hf && hf->var_size)) {
7268                 status = -EINVAL;
7269                 goto end;
7270         }
7271
7272         offset[0] = h0 ? hf->offset : mf->offset;
7273
7274         for (i = 1; i < params->n_fields; i++)
7275                 if (h0) {
7276                         struct header *h;
7277
7278                         hf = header_field_parse(p, params->fields[i].name, &h);
7279                         if (!hf || (h->id != h0->id) || hf->var_size) {
7280                                 status = -EINVAL;
7281                                 goto end;
7282                         }
7283
7284                         offset[i] = hf->offset;
7285                 } else {
7286                         mf = metadata_field_parse(p, params->fields[i].name);
7287                         if (!mf) {
7288                                 status = -EINVAL;
7289                                 goto end;
7290                         }
7291
7292                         offset[i] = mf->offset;
7293                 }
7294
7295         /* Check that there are no duplicated match fields. */
7296         for (i = 0; i < params->n_fields; i++) {
7297                 uint32_t j;
7298
7299                 for (j = 0; j < i; j++)
7300                         if (offset[j] == offset[i]) {
7301                                 status = -EINVAL;
7302                                 goto end;
7303                         }
7304         }
7305
7306         /* Return. */
7307         if (header)
7308                 *header = h0;
7309
7310 end:
7311         free(offset);
7312         return status;
7313 }
7314
7315 int
7316 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7317                               const char *name,
7318                               struct rte_swx_pipeline_table_params *params,
7319                               const char *recommended_table_type_name,
7320                               const char *args,
7321                               uint32_t size)
7322 {
7323         struct table_type *type;
7324         struct table *t = NULL;
7325         struct action *default_action;
7326         struct header *header = NULL;
7327         uint32_t action_data_size_max = 0, i;
7328         int status = 0;
7329
7330         CHECK(p, EINVAL);
7331
7332         CHECK_NAME(name, EINVAL);
7333         CHECK(!table_find(p, name), EEXIST);
7334         CHECK(!selector_find(p, name), EEXIST);
7335         CHECK(!learner_find(p, name), EEXIST);
7336
7337         CHECK(params, EINVAL);
7338
7339         /* Match checks. */
7340         status = table_match_fields_check(p, params, &header);
7341         if (status)
7342                 return status;
7343
7344         /* Action checks. */
7345         CHECK(params->n_actions, EINVAL);
7346         CHECK(params->action_names, EINVAL);
7347         for (i = 0; i < params->n_actions; i++) {
7348                 const char *action_name = params->action_names[i];
7349                 struct action *a;
7350                 uint32_t action_data_size;
7351                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7352
7353                 CHECK_NAME(action_name, EINVAL);
7354
7355                 a = action_find(p, action_name);
7356                 CHECK(a, EINVAL);
7357                 CHECK(!action_does_learning(a), EINVAL);
7358
7359                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7360                 if (action_data_size > action_data_size_max)
7361                         action_data_size_max = action_data_size;
7362
7363                 if (params->action_is_for_table_entries)
7364                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7365                 if (params->action_is_for_default_entry)
7366                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7367                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7368         }
7369
7370         CHECK_NAME(params->default_action_name, EINVAL);
7371         for (i = 0; i < p->n_actions; i++)
7372                 if (!strcmp(params->action_names[i],
7373                             params->default_action_name))
7374                         break;
7375         CHECK(i < params->n_actions, EINVAL);
7376         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7377               EINVAL);
7378
7379         default_action = action_find(p, params->default_action_name);
7380         CHECK((default_action->st && params->default_action_data) ||
7381               !params->default_action_data, EINVAL);
7382
7383         /* Table type checks. */
7384         if (recommended_table_type_name)
7385                 CHECK_NAME(recommended_table_type_name, EINVAL);
7386
7387         if (params->n_fields) {
7388                 enum rte_swx_table_match_type match_type;
7389
7390                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7391                 if (status)
7392                         return status;
7393
7394                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7395                 CHECK(type, EINVAL);
7396         } else {
7397                 type = NULL;
7398         }
7399
7400         /* Memory allocation. */
7401         t = calloc(1, sizeof(struct table));
7402         if (!t)
7403                 goto nomem;
7404
7405         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7406         if (!t->fields)
7407                 goto nomem;
7408
7409         t->actions = calloc(params->n_actions, sizeof(struct action *));
7410         if (!t->actions)
7411                 goto nomem;
7412
7413         if (action_data_size_max) {
7414                 t->default_action_data = calloc(1, action_data_size_max);
7415                 if (!t->default_action_data)
7416                         goto nomem;
7417         }
7418
7419         t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7420         if (!t->action_is_for_table_entries)
7421                 goto nomem;
7422
7423         t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7424         if (!t->action_is_for_default_entry)
7425                 goto nomem;
7426
7427         /* Node initialization. */
7428         strcpy(t->name, name);
7429         if (args && args[0])
7430                 strcpy(t->args, args);
7431         t->type = type;
7432
7433         for (i = 0; i < params->n_fields; i++) {
7434                 struct rte_swx_match_field_params *field = &params->fields[i];
7435                 struct match_field *f = &t->fields[i];
7436
7437                 f->match_type = field->match_type;
7438                 f->field = header ?
7439                         header_field_parse(p, field->name, NULL) :
7440                         metadata_field_parse(p, field->name);
7441         }
7442         t->n_fields = params->n_fields;
7443         t->header = header;
7444
7445         for (i = 0; i < params->n_actions; i++) {
7446                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7447
7448                 if (params->action_is_for_table_entries)
7449                         action_is_for_table_entries = params->action_is_for_table_entries[i];
7450                 if (params->action_is_for_default_entry)
7451                         action_is_for_default_entry = params->action_is_for_default_entry[i];
7452
7453                 t->actions[i] = action_find(p, params->action_names[i]);
7454                 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7455                 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7456         }
7457         t->default_action = default_action;
7458         if (default_action->st)
7459                 memcpy(t->default_action_data,
7460                        params->default_action_data,
7461                        default_action->st->n_bits / 8);
7462         t->n_actions = params->n_actions;
7463         t->default_action_is_const = params->default_action_is_const;
7464         t->action_data_size_max = action_data_size_max;
7465
7466         t->size = size;
7467         t->id = p->n_tables;
7468
7469         /* Node add to tailq. */
7470         TAILQ_INSERT_TAIL(&p->tables, t, node);
7471         p->n_tables++;
7472
7473         return 0;
7474
7475 nomem:
7476         if (!t)
7477                 return -ENOMEM;
7478
7479         free(t->action_is_for_default_entry);
7480         free(t->action_is_for_table_entries);
7481         free(t->default_action_data);
7482         free(t->actions);
7483         free(t->fields);
7484         free(t);
7485
7486         return -ENOMEM;
7487 }
7488
7489 static struct rte_swx_table_params *
7490 table_params_get(struct table *table)
7491 {
7492         struct rte_swx_table_params *params;
7493         struct field *first, *last;
7494         uint8_t *key_mask;
7495         uint32_t key_size, key_offset, action_data_size, i;
7496
7497         /* Memory allocation. */
7498         params = calloc(1, sizeof(struct rte_swx_table_params));
7499         if (!params)
7500                 return NULL;
7501
7502         /* Find first (smallest offset) and last (biggest offset) match fields. */
7503         first = table->fields[0].field;
7504         last = table->fields[0].field;
7505
7506         for (i = 0; i < table->n_fields; i++) {
7507                 struct field *f = table->fields[i].field;
7508
7509                 if (f->offset < first->offset)
7510                         first = f;
7511
7512                 if (f->offset > last->offset)
7513                         last = f;
7514         }
7515
7516         /* Key offset and size. */
7517         key_offset = first->offset / 8;
7518         key_size = (last->offset + last->n_bits - first->offset) / 8;
7519
7520         /* Memory allocation. */
7521         key_mask = calloc(1, key_size);
7522         if (!key_mask) {
7523                 free(params);
7524                 return NULL;
7525         }
7526
7527         /* Key mask. */
7528         for (i = 0; i < table->n_fields; i++) {
7529                 struct field *f = table->fields[i].field;
7530                 uint32_t start = (f->offset - first->offset) / 8;
7531                 size_t size = f->n_bits / 8;
7532
7533                 memset(&key_mask[start], 0xFF, size);
7534         }
7535
7536         /* Action data size. */
7537         action_data_size = 0;
7538         for (i = 0; i < table->n_actions; i++) {
7539                 struct action *action = table->actions[i];
7540                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7541
7542                 if (ads > action_data_size)
7543                         action_data_size = ads;
7544         }
7545
7546         /* Fill in. */
7547         params->match_type = table->type->match_type;
7548         params->key_size = key_size;
7549         params->key_offset = key_offset;
7550         params->key_mask0 = key_mask;
7551         params->action_data_size = action_data_size;
7552         params->n_keys_max = table->size;
7553
7554         return params;
7555 }
7556
7557 static void
7558 table_params_free(struct rte_swx_table_params *params)
7559 {
7560         if (!params)
7561                 return;
7562
7563         free(params->key_mask0);
7564         free(params);
7565 }
7566
7567 static int
7568 table_stub_lkp(void *table __rte_unused,
7569                void *mailbox __rte_unused,
7570                uint8_t **key __rte_unused,
7571                uint64_t *action_id __rte_unused,
7572                uint8_t **action_data __rte_unused,
7573                int *hit)
7574 {
7575         *hit = 0;
7576         return 1; /* DONE. */
7577 }
7578
7579 static int
7580 table_build(struct rte_swx_pipeline *p)
7581 {
7582         uint32_t i;
7583
7584         /* Per pipeline: table statistics. */
7585         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7586         CHECK(p->table_stats, ENOMEM);
7587
7588         for (i = 0; i < p->n_tables; i++) {
7589                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7590                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7591         }
7592
7593         /* Per thread: table runt-time. */
7594         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7595                 struct thread *t = &p->threads[i];
7596                 struct table *table;
7597
7598                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7599                 CHECK(t->tables, ENOMEM);
7600
7601                 TAILQ_FOREACH(table, &p->tables, node) {
7602                         struct table_runtime *r = &t->tables[table->id];
7603
7604                         if (table->type) {
7605                                 uint64_t size;
7606
7607                                 size = table->type->ops.mailbox_size_get();
7608
7609                                 /* r->func. */
7610                                 r->func = table->type->ops.lkp;
7611
7612                                 /* r->mailbox. */
7613                                 if (size) {
7614                                         r->mailbox = calloc(1, size);
7615                                         CHECK(r->mailbox, ENOMEM);
7616                                 }
7617
7618                                 /* r->key. */
7619                                 r->key = table->header ?
7620                                         &t->structs[table->header->struct_id] :
7621                                         &t->structs[p->metadata_struct_id];
7622                         } else {
7623                                 r->func = table_stub_lkp;
7624                         }
7625                 }
7626         }
7627
7628         return 0;
7629 }
7630
7631 static void
7632 table_build_free(struct rte_swx_pipeline *p)
7633 {
7634         uint32_t i;
7635
7636         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7637                 struct thread *t = &p->threads[i];
7638                 uint32_t j;
7639
7640                 if (!t->tables)
7641                         continue;
7642
7643                 for (j = 0; j < p->n_tables; j++) {
7644                         struct table_runtime *r = &t->tables[j];
7645
7646                         free(r->mailbox);
7647                 }
7648
7649                 free(t->tables);
7650                 t->tables = NULL;
7651         }
7652
7653         if (p->table_stats) {
7654                 for (i = 0; i < p->n_tables; i++)
7655                         free(p->table_stats[i].n_pkts_action);
7656
7657                 free(p->table_stats);
7658         }
7659 }
7660
7661 static void
7662 table_free(struct rte_swx_pipeline *p)
7663 {
7664         table_build_free(p);
7665
7666         /* Tables. */
7667         for ( ; ; ) {
7668                 struct table *elem;
7669
7670                 elem = TAILQ_FIRST(&p->tables);
7671                 if (!elem)
7672                         break;
7673
7674                 TAILQ_REMOVE(&p->tables, elem, node);
7675                 free(elem->fields);
7676                 free(elem->actions);
7677                 free(elem->default_action_data);
7678                 free(elem);
7679         }
7680
7681         /* Table types. */
7682         for ( ; ; ) {
7683                 struct table_type *elem;
7684
7685                 elem = TAILQ_FIRST(&p->table_types);
7686                 if (!elem)
7687                         break;
7688
7689                 TAILQ_REMOVE(&p->table_types, elem, node);
7690                 free(elem);
7691         }
7692 }
7693
7694 /*
7695  * Selector.
7696  */
7697 static struct selector *
7698 selector_find(struct rte_swx_pipeline *p, const char *name)
7699 {
7700         struct selector *s;
7701
7702         TAILQ_FOREACH(s, &p->selectors, node)
7703                 if (strcmp(s->name, name) == 0)
7704                         return s;
7705
7706         return NULL;
7707 }
7708
7709 static struct selector *
7710 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7711 {
7712         struct selector *s = NULL;
7713
7714         TAILQ_FOREACH(s, &p->selectors, node)
7715                 if (s->id == id)
7716                         return s;
7717
7718         return NULL;
7719 }
7720
7721 static int
7722 selector_fields_check(struct rte_swx_pipeline *p,
7723                       struct rte_swx_pipeline_selector_params *params,
7724                       struct header **header)
7725 {
7726         struct header *h0 = NULL;
7727         struct field *hf, *mf;
7728         uint32_t i;
7729
7730         /* Return if no selector fields. */
7731         if (!params->n_selector_fields || !params->selector_field_names)
7732                 return -EINVAL;
7733
7734         /* Check that all the selector fields either belong to the same header
7735          * or are all meta-data fields.
7736          */
7737         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7738         mf = metadata_field_parse(p, params->selector_field_names[0]);
7739         if (!hf && !mf)
7740                 return -EINVAL;
7741
7742         for (i = 1; i < params->n_selector_fields; i++)
7743                 if (h0) {
7744                         struct header *h;
7745
7746                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7747                         if (!hf || (h->id != h0->id))
7748                                 return -EINVAL;
7749                 } else {
7750                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7751                         if (!mf)
7752                                 return -EINVAL;
7753                 }
7754
7755         /* Check that there are no duplicated match fields. */
7756         for (i = 0; i < params->n_selector_fields; i++) {
7757                 const char *field_name = params->selector_field_names[i];
7758                 uint32_t j;
7759
7760                 for (j = i + 1; j < params->n_selector_fields; j++)
7761                         if (!strcmp(params->selector_field_names[j], field_name))
7762                                 return -EINVAL;
7763         }
7764
7765         /* Return. */
7766         if (header)
7767                 *header = h0;
7768
7769         return 0;
7770 }
7771
7772 int
7773 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7774                                  const char *name,
7775                                  struct rte_swx_pipeline_selector_params *params)
7776 {
7777         struct selector *s;
7778         struct header *selector_header = NULL;
7779         struct field *group_id_field, *member_id_field;
7780         uint32_t i;
7781         int status = 0;
7782
7783         CHECK(p, EINVAL);
7784
7785         CHECK_NAME(name, EINVAL);
7786         CHECK(!table_find(p, name), EEXIST);
7787         CHECK(!selector_find(p, name), EEXIST);
7788         CHECK(!learner_find(p, name), EEXIST);
7789
7790         CHECK(params, EINVAL);
7791
7792         CHECK_NAME(params->group_id_field_name, EINVAL);
7793         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7794         CHECK(group_id_field, EINVAL);
7795
7796         for (i = 0; i < params->n_selector_fields; i++) {
7797                 const char *field_name = params->selector_field_names[i];
7798
7799                 CHECK_NAME(field_name, EINVAL);
7800         }
7801         status = selector_fields_check(p, params, &selector_header);
7802         if (status)
7803                 return status;
7804
7805         CHECK_NAME(params->member_id_field_name, EINVAL);
7806         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7807         CHECK(member_id_field, EINVAL);
7808
7809         CHECK(params->n_groups_max, EINVAL);
7810
7811         CHECK(params->n_members_per_group_max, EINVAL);
7812
7813         /* Memory allocation. */
7814         s = calloc(1, sizeof(struct selector));
7815         if (!s) {
7816                 status = -ENOMEM;
7817                 goto error;
7818         }
7819
7820         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7821         if (!s->selector_fields) {
7822                 status = -ENOMEM;
7823                 goto error;
7824         }
7825
7826         /* Node initialization. */
7827         strcpy(s->name, name);
7828
7829         s->group_id_field = group_id_field;
7830
7831         for (i = 0; i < params->n_selector_fields; i++) {
7832                 const char *field_name = params->selector_field_names[i];
7833
7834                 s->selector_fields[i] = selector_header ?
7835                         header_field_parse(p, field_name, NULL) :
7836                         metadata_field_parse(p, field_name);
7837         }
7838
7839         s->n_selector_fields = params->n_selector_fields;
7840
7841         s->selector_header = selector_header;
7842
7843         s->member_id_field = member_id_field;
7844
7845         s->n_groups_max = params->n_groups_max;
7846
7847         s->n_members_per_group_max = params->n_members_per_group_max;
7848
7849         s->id = p->n_selectors;
7850
7851         /* Node add to tailq. */
7852         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7853         p->n_selectors++;
7854
7855         return 0;
7856
7857 error:
7858         if (!s)
7859                 return status;
7860
7861         free(s->selector_fields);
7862
7863         free(s);
7864
7865         return status;
7866 }
7867
7868 static void
7869 selector_params_free(struct rte_swx_table_selector_params *params)
7870 {
7871         if (!params)
7872                 return;
7873
7874         free(params->selector_mask);
7875
7876         free(params);
7877 }
7878
7879 static struct rte_swx_table_selector_params *
7880 selector_table_params_get(struct selector *s)
7881 {
7882         struct rte_swx_table_selector_params *params = NULL;
7883         struct field *first, *last;
7884         uint32_t i;
7885
7886         /* Memory allocation. */
7887         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7888         if (!params)
7889                 goto error;
7890
7891         /* Group ID. */
7892         params->group_id_offset = s->group_id_field->offset / 8;
7893
7894         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7895         first = s->selector_fields[0];
7896         last = s->selector_fields[0];
7897
7898         for (i = 0; i < s->n_selector_fields; i++) {
7899                 struct field *f = s->selector_fields[i];
7900
7901                 if (f->offset < first->offset)
7902                         first = f;
7903
7904                 if (f->offset > last->offset)
7905                         last = f;
7906         }
7907
7908         /* Selector offset and size. */
7909         params->selector_offset = first->offset / 8;
7910         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7911
7912         /* Memory allocation. */
7913         params->selector_mask = calloc(1, params->selector_size);
7914         if (!params->selector_mask)
7915                 goto error;
7916
7917         /* Selector mask. */
7918         for (i = 0; i < s->n_selector_fields; i++) {
7919                 struct field *f = s->selector_fields[i];
7920                 uint32_t start = (f->offset - first->offset) / 8;
7921                 size_t size = f->n_bits / 8;
7922
7923                 memset(&params->selector_mask[start], 0xFF, size);
7924         }
7925
7926         /* Member ID. */
7927         params->member_id_offset = s->member_id_field->offset / 8;
7928
7929         /* Maximum number of groups. */
7930         params->n_groups_max = s->n_groups_max;
7931
7932         /* Maximum number of members per group. */
7933         params->n_members_per_group_max = s->n_members_per_group_max;
7934
7935         return params;
7936
7937 error:
7938         selector_params_free(params);
7939         return NULL;
7940 }
7941
7942 static void
7943 selector_build_free(struct rte_swx_pipeline *p)
7944 {
7945         uint32_t i;
7946
7947         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7948                 struct thread *t = &p->threads[i];
7949                 uint32_t j;
7950
7951                 if (!t->selectors)
7952                         continue;
7953
7954                 for (j = 0; j < p->n_selectors; j++) {
7955                         struct selector_runtime *r = &t->selectors[j];
7956
7957                         free(r->mailbox);
7958                 }
7959
7960                 free(t->selectors);
7961                 t->selectors = NULL;
7962         }
7963
7964         free(p->selector_stats);
7965         p->selector_stats = NULL;
7966 }
7967
7968 static int
7969 selector_build(struct rte_swx_pipeline *p)
7970 {
7971         uint32_t i;
7972         int status = 0;
7973
7974         /* Per pipeline: selector statistics. */
7975         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7976         if (!p->selector_stats) {
7977                 status = -ENOMEM;
7978                 goto error;
7979         }
7980
7981         /* Per thread: selector run-time. */
7982         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7983                 struct thread *t = &p->threads[i];
7984                 struct selector *s;
7985
7986                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7987                 if (!t->selectors) {
7988                         status = -ENOMEM;
7989                         goto error;
7990                 }
7991
7992                 TAILQ_FOREACH(s, &p->selectors, node) {
7993                         struct selector_runtime *r = &t->selectors[s->id];
7994                         uint64_t size;
7995
7996                         /* r->mailbox. */
7997                         size = rte_swx_table_selector_mailbox_size_get();
7998                         if (size) {
7999                                 r->mailbox = calloc(1, size);
8000                                 if (!r->mailbox) {
8001                                         status = -ENOMEM;
8002                                         goto error;
8003                                 }
8004                         }
8005
8006                         /* r->group_id_buffer. */
8007                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
8008
8009                         /* r->selector_buffer. */
8010                         r->selector_buffer = s->selector_header ?
8011                                 &t->structs[s->selector_header->struct_id] :
8012                                 &t->structs[p->metadata_struct_id];
8013
8014                         /* r->member_id_buffer. */
8015                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
8016                 }
8017         }
8018
8019         return 0;
8020
8021 error:
8022         selector_build_free(p);
8023         return status;
8024 }
8025
8026 static void
8027 selector_free(struct rte_swx_pipeline *p)
8028 {
8029         selector_build_free(p);
8030
8031         /* Selector tables. */
8032         for ( ; ; ) {
8033                 struct selector *elem;
8034
8035                 elem = TAILQ_FIRST(&p->selectors);
8036                 if (!elem)
8037                         break;
8038
8039                 TAILQ_REMOVE(&p->selectors, elem, node);
8040                 free(elem->selector_fields);
8041                 free(elem);
8042         }
8043 }
8044
8045 /*
8046  * Learner table.
8047  */
8048 static struct learner *
8049 learner_find(struct rte_swx_pipeline *p, const char *name)
8050 {
8051         struct learner *l;
8052
8053         TAILQ_FOREACH(l, &p->learners, node)
8054                 if (!strcmp(l->name, name))
8055                         return l;
8056
8057         return NULL;
8058 }
8059
8060 static struct learner *
8061 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8062 {
8063         struct learner *l = NULL;
8064
8065         TAILQ_FOREACH(l, &p->learners, node)
8066                 if (l->id == id)
8067                         return l;
8068
8069         return NULL;
8070 }
8071
8072 static int
8073 learner_match_fields_check(struct rte_swx_pipeline *p,
8074                            struct rte_swx_pipeline_learner_params *params,
8075                            struct header **header)
8076 {
8077         struct header *h0 = NULL;
8078         struct field *hf, *mf;
8079         uint32_t i;
8080
8081         /* Return if no match fields. */
8082         if (!params->n_fields || !params->field_names)
8083                 return -EINVAL;
8084
8085         /* Check that all the match fields either belong to the same header
8086          * or are all meta-data fields.
8087          */
8088         hf = header_field_parse(p, params->field_names[0], &h0);
8089         mf = metadata_field_parse(p, params->field_names[0]);
8090         if (!hf && !mf)
8091                 return -EINVAL;
8092
8093         for (i = 1; i < params->n_fields; i++)
8094                 if (h0) {
8095                         struct header *h;
8096
8097                         hf = header_field_parse(p, params->field_names[i], &h);
8098                         if (!hf || (h->id != h0->id))
8099                                 return -EINVAL;
8100                 } else {
8101                         mf = metadata_field_parse(p, params->field_names[i]);
8102                         if (!mf)
8103                                 return -EINVAL;
8104                 }
8105
8106         /* Check that there are no duplicated match fields. */
8107         for (i = 0; i < params->n_fields; i++) {
8108                 const char *field_name = params->field_names[i];
8109                 uint32_t j;
8110
8111                 for (j = i + 1; j < params->n_fields; j++)
8112                         if (!strcmp(params->field_names[j], field_name))
8113                                 return -EINVAL;
8114         }
8115
8116         /* Return. */
8117         if (header)
8118                 *header = h0;
8119
8120         return 0;
8121 }
8122
8123 static int
8124 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8125 {
8126         struct struct_type *mst = p->metadata_st, *ast = a->st;
8127         struct field *mf, *af;
8128         uint32_t mf_pos, i;
8129
8130         if (!ast) {
8131                 if (mf_name)
8132                         return -EINVAL;
8133
8134                 return 0;
8135         }
8136
8137         /* Check that mf_name is the name of a valid meta-data field. */
8138         CHECK_NAME(mf_name, EINVAL);
8139         mf = metadata_field_parse(p, mf_name);
8140         CHECK(mf, EINVAL);
8141
8142         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8143          * all the action arguments.
8144          */
8145         mf_pos = mf - mst->fields;
8146         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8147
8148         /* Check that the size of each of the identified meta-data fields matches exactly the size
8149          * of the corresponding action argument.
8150          */
8151         for (i = 0; i < ast->n_fields; i++) {
8152                 mf = &mst->fields[mf_pos + i];
8153                 af = &ast->fields[i];
8154
8155                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8156         }
8157
8158         return 0;
8159 }
8160
8161 static int
8162 learner_action_learning_check(struct rte_swx_pipeline *p,
8163                               struct action *action,
8164                               const char **action_names,
8165                               uint32_t n_actions)
8166 {
8167         uint32_t i;
8168
8169         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8170          * the action passed as argument to the "learn" instruction) is also enabled for the
8171          * current learner table.
8172          */
8173         for (i = 0; i < action->n_instructions; i++) {
8174                 struct instruction *instr = &action->instructions[i];
8175                 uint32_t found = 0, j;
8176
8177                 if (instr->type != INSTR_LEARNER_LEARN)
8178                         continue;
8179
8180                 for (j = 0; j < n_actions; j++) {
8181                         struct action *a;
8182
8183                         a = action_find(p, action_names[j]);
8184                         if (!a)
8185                                 return -EINVAL;
8186
8187                         if (a->id == instr->learn.action_id)
8188                                 found = 1;
8189                 }
8190
8191                 if (!found)
8192                         return -EINVAL;
8193         }
8194
8195         return 0;
8196 }
8197
8198 int
8199 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8200                               const char *name,
8201                               struct rte_swx_pipeline_learner_params *params,
8202                               uint32_t size,
8203                               uint32_t timeout)
8204 {
8205         struct learner *l = NULL;
8206         struct action *default_action;
8207         struct header *header = NULL;
8208         uint32_t action_data_size_max = 0, i;
8209         int status = 0;
8210
8211         CHECK(p, EINVAL);
8212
8213         CHECK_NAME(name, EINVAL);
8214         CHECK(!table_find(p, name), EEXIST);
8215         CHECK(!selector_find(p, name), EEXIST);
8216         CHECK(!learner_find(p, name), EEXIST);
8217
8218         CHECK(params, EINVAL);
8219
8220         /* Match checks. */
8221         status = learner_match_fields_check(p, params, &header);
8222         if (status)
8223                 return status;
8224
8225         /* Action checks. */
8226         CHECK(params->n_actions, EINVAL);
8227         CHECK(params->action_names, EINVAL);
8228         for (i = 0; i < params->n_actions; i++) {
8229                 const char *action_name = params->action_names[i];
8230                 struct action *a;
8231                 uint32_t action_data_size;
8232                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8233
8234                 CHECK_NAME(action_name, EINVAL);
8235
8236                 a = action_find(p, action_name);
8237                 CHECK(a, EINVAL);
8238
8239                 status = learner_action_learning_check(p,
8240                                                        a,
8241                                                        params->action_names,
8242                                                        params->n_actions);
8243                 if (status)
8244                         return status;
8245
8246                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8247                 if (action_data_size > action_data_size_max)
8248                         action_data_size_max = action_data_size;
8249
8250                 if (params->action_is_for_table_entries)
8251                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8252                 if (params->action_is_for_default_entry)
8253                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8254                 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8255         }
8256
8257         CHECK_NAME(params->default_action_name, EINVAL);
8258         for (i = 0; i < p->n_actions; i++)
8259                 if (!strcmp(params->action_names[i],
8260                             params->default_action_name))
8261                         break;
8262         CHECK(i < params->n_actions, EINVAL);
8263         CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8264               EINVAL);
8265
8266         default_action = action_find(p, params->default_action_name);
8267         CHECK((default_action->st && params->default_action_data) ||
8268               !params->default_action_data, EINVAL);
8269
8270         /* Any other checks. */
8271         CHECK(size, EINVAL);
8272         CHECK(timeout, EINVAL);
8273
8274         /* Memory allocation. */
8275         l = calloc(1, sizeof(struct learner));
8276         if (!l)
8277                 goto nomem;
8278
8279         l->fields = calloc(params->n_fields, sizeof(struct field *));
8280         if (!l->fields)
8281                 goto nomem;
8282
8283         l->actions = calloc(params->n_actions, sizeof(struct action *));
8284         if (!l->actions)
8285                 goto nomem;
8286
8287         if (action_data_size_max) {
8288                 l->default_action_data = calloc(1, action_data_size_max);
8289                 if (!l->default_action_data)
8290                         goto nomem;
8291         }
8292
8293         l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8294         if (!l->action_is_for_table_entries)
8295                 goto nomem;
8296
8297         l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8298         if (!l->action_is_for_default_entry)
8299                 goto nomem;
8300
8301         /* Node initialization. */
8302         strcpy(l->name, name);
8303
8304         for (i = 0; i < params->n_fields; i++) {
8305                 const char *field_name = params->field_names[i];
8306
8307                 l->fields[i] = header ?
8308                         header_field_parse(p, field_name, NULL) :
8309                         metadata_field_parse(p, field_name);
8310         }
8311
8312         l->n_fields = params->n_fields;
8313
8314         l->header = header;
8315
8316         for (i = 0; i < params->n_actions; i++) {
8317                 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8318
8319                 if (params->action_is_for_table_entries)
8320                         action_is_for_table_entries = params->action_is_for_table_entries[i];
8321                 if (params->action_is_for_default_entry)
8322                         action_is_for_default_entry = params->action_is_for_default_entry[i];
8323
8324                 l->actions[i] = action_find(p, params->action_names[i]);
8325                 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8326                 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8327         }
8328
8329         l->default_action = default_action;
8330
8331         if (default_action->st)
8332                 memcpy(l->default_action_data,
8333                        params->default_action_data,
8334                        default_action->st->n_bits / 8);
8335
8336         l->n_actions = params->n_actions;
8337
8338         l->default_action_is_const = params->default_action_is_const;
8339
8340         l->action_data_size_max = action_data_size_max;
8341
8342         l->size = size;
8343
8344         l->timeout = timeout;
8345
8346         l->id = p->n_learners;
8347
8348         /* Node add to tailq. */
8349         TAILQ_INSERT_TAIL(&p->learners, l, node);
8350         p->n_learners++;
8351
8352         return 0;
8353
8354 nomem:
8355         if (!l)
8356                 return -ENOMEM;
8357
8358         free(l->action_is_for_default_entry);
8359         free(l->action_is_for_table_entries);
8360         free(l->default_action_data);
8361         free(l->actions);
8362         free(l->fields);
8363         free(l);
8364
8365         return -ENOMEM;
8366 }
8367
8368 static void
8369 learner_params_free(struct rte_swx_table_learner_params *params)
8370 {
8371         if (!params)
8372                 return;
8373
8374         free(params->key_mask0);
8375
8376         free(params);
8377 }
8378
8379 static struct rte_swx_table_learner_params *
8380 learner_params_get(struct learner *l)
8381 {
8382         struct rte_swx_table_learner_params *params = NULL;
8383         struct field *first, *last;
8384         uint32_t i;
8385
8386         /* Memory allocation. */
8387         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8388         if (!params)
8389                 goto error;
8390
8391         /* Find first (smallest offset) and last (biggest offset) match fields. */
8392         first = l->fields[0];
8393         last = l->fields[0];
8394
8395         for (i = 0; i < l->n_fields; i++) {
8396                 struct field *f = l->fields[i];
8397
8398                 if (f->offset < first->offset)
8399                         first = f;
8400
8401                 if (f->offset > last->offset)
8402                         last = f;
8403         }
8404
8405         /* Key offset and size. */
8406         params->key_offset = first->offset / 8;
8407         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8408
8409         /* Memory allocation. */
8410         params->key_mask0 = calloc(1, params->key_size);
8411         if (!params->key_mask0)
8412                 goto error;
8413
8414         /* Key mask. */
8415         for (i = 0; i < l->n_fields; i++) {
8416                 struct field *f = l->fields[i];
8417                 uint32_t start = (f->offset - first->offset) / 8;
8418                 size_t size = f->n_bits / 8;
8419
8420                 memset(&params->key_mask0[start], 0xFF, size);
8421         }
8422
8423         /* Action data size. */
8424         params->action_data_size = l->action_data_size_max;
8425
8426         /* Maximum number of keys. */
8427         params->n_keys_max = l->size;
8428
8429         /* Timeout. */
8430         params->key_timeout = l->timeout;
8431
8432         return params;
8433
8434 error:
8435         learner_params_free(params);
8436         return NULL;
8437 }
8438
8439 static void
8440 learner_build_free(struct rte_swx_pipeline *p)
8441 {
8442         uint32_t i;
8443
8444         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8445                 struct thread *t = &p->threads[i];
8446                 uint32_t j;
8447
8448                 if (!t->learners)
8449                         continue;
8450
8451                 for (j = 0; j < p->n_learners; j++) {
8452                         struct learner_runtime *r = &t->learners[j];
8453
8454                         free(r->mailbox);
8455                 }
8456
8457                 free(t->learners);
8458                 t->learners = NULL;
8459         }
8460
8461         if (p->learner_stats) {
8462                 for (i = 0; i < p->n_learners; i++)
8463                         free(p->learner_stats[i].n_pkts_action);
8464
8465                 free(p->learner_stats);
8466         }
8467 }
8468
8469 static int
8470 learner_build(struct rte_swx_pipeline *p)
8471 {
8472         uint32_t i;
8473         int status = 0;
8474
8475         /* Per pipeline: learner statistics. */
8476         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8477         CHECK(p->learner_stats, ENOMEM);
8478
8479         for (i = 0; i < p->n_learners; i++) {
8480                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8481                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8482         }
8483
8484         /* Per thread: learner run-time. */
8485         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8486                 struct thread *t = &p->threads[i];
8487                 struct learner *l;
8488
8489                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8490                 if (!t->learners) {
8491                         status = -ENOMEM;
8492                         goto error;
8493                 }
8494
8495                 TAILQ_FOREACH(l, &p->learners, node) {
8496                         struct learner_runtime *r = &t->learners[l->id];
8497                         uint64_t size;
8498
8499                         /* r->mailbox. */
8500                         size = rte_swx_table_learner_mailbox_size_get();
8501                         if (size) {
8502                                 r->mailbox = calloc(1, size);
8503                                 if (!r->mailbox) {
8504                                         status = -ENOMEM;
8505                                         goto error;
8506                                 }
8507                         }
8508
8509                         /* r->key. */
8510                         r->key = l->header ?
8511                                 &t->structs[l->header->struct_id] :
8512                                 &t->structs[p->metadata_struct_id];
8513                 }
8514         }
8515
8516         return 0;
8517
8518 error:
8519         learner_build_free(p);
8520         return status;
8521 }
8522
8523 static void
8524 learner_free(struct rte_swx_pipeline *p)
8525 {
8526         learner_build_free(p);
8527
8528         /* Learner tables. */
8529         for ( ; ; ) {
8530                 struct learner *l;
8531
8532                 l = TAILQ_FIRST(&p->learners);
8533                 if (!l)
8534                         break;
8535
8536                 TAILQ_REMOVE(&p->learners, l, node);
8537                 free(l->fields);
8538                 free(l->actions);
8539                 free(l->default_action_data);
8540                 free(l);
8541         }
8542 }
8543
8544 /*
8545  * Table state.
8546  */
8547 static int
8548 table_state_build(struct rte_swx_pipeline *p)
8549 {
8550         struct table *table;
8551         struct selector *s;
8552         struct learner *l;
8553
8554         p->table_state = calloc(p->n_tables + p->n_selectors,
8555                                 sizeof(struct rte_swx_table_state));
8556         CHECK(p->table_state, ENOMEM);
8557
8558         TAILQ_FOREACH(table, &p->tables, node) {
8559                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8560
8561                 if (table->type) {
8562                         struct rte_swx_table_params *params;
8563
8564                         /* ts->obj. */
8565                         params = table_params_get(table);
8566                         CHECK(params, ENOMEM);
8567
8568                         ts->obj = table->type->ops.create(params,
8569                                 NULL,
8570                                 table->args,
8571                                 p->numa_node);
8572
8573                         table_params_free(params);
8574                         CHECK(ts->obj, ENODEV);
8575                 }
8576
8577                 /* ts->default_action_data. */
8578                 if (table->action_data_size_max) {
8579                         ts->default_action_data =
8580                                 malloc(table->action_data_size_max);
8581                         CHECK(ts->default_action_data, ENOMEM);
8582
8583                         memcpy(ts->default_action_data,
8584                                table->default_action_data,
8585                                table->action_data_size_max);
8586                 }
8587
8588                 /* ts->default_action_id. */
8589                 ts->default_action_id = table->default_action->id;
8590         }
8591
8592         TAILQ_FOREACH(s, &p->selectors, node) {
8593                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8594                 struct rte_swx_table_selector_params *params;
8595
8596                 /* ts->obj. */
8597                 params = selector_table_params_get(s);
8598                 CHECK(params, ENOMEM);
8599
8600                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8601
8602                 selector_params_free(params);
8603                 CHECK(ts->obj, ENODEV);
8604         }
8605
8606         TAILQ_FOREACH(l, &p->learners, node) {
8607                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8608                         p->n_selectors + l->id];
8609                 struct rte_swx_table_learner_params *params;
8610
8611                 /* ts->obj. */
8612                 params = learner_params_get(l);
8613                 CHECK(params, ENOMEM);
8614
8615                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8616                 learner_params_free(params);
8617                 CHECK(ts->obj, ENODEV);
8618
8619                 /* ts->default_action_data. */
8620                 if (l->action_data_size_max) {
8621                         ts->default_action_data = malloc(l->action_data_size_max);
8622                         CHECK(ts->default_action_data, ENOMEM);
8623
8624                         memcpy(ts->default_action_data,
8625                                l->default_action_data,
8626                                l->action_data_size_max);
8627                 }
8628
8629                 /* ts->default_action_id. */
8630                 ts->default_action_id = l->default_action->id;
8631         }
8632
8633         return 0;
8634 }
8635
8636 static void
8637 table_state_build_free(struct rte_swx_pipeline *p)
8638 {
8639         uint32_t i;
8640
8641         if (!p->table_state)
8642                 return;
8643
8644         for (i = 0; i < p->n_tables; i++) {
8645                 struct rte_swx_table_state *ts = &p->table_state[i];
8646                 struct table *table = table_find_by_id(p, i);
8647
8648                 /* ts->obj. */
8649                 if (table->type && ts->obj)
8650                         table->type->ops.free(ts->obj);
8651
8652                 /* ts->default_action_data. */
8653                 free(ts->default_action_data);
8654         }
8655
8656         for (i = 0; i < p->n_selectors; i++) {
8657                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8658
8659                 /* ts->obj. */
8660                 if (ts->obj)
8661                         rte_swx_table_selector_free(ts->obj);
8662         }
8663
8664         for (i = 0; i < p->n_learners; i++) {
8665                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8666
8667                 /* ts->obj. */
8668                 if (ts->obj)
8669                         rte_swx_table_learner_free(ts->obj);
8670
8671                 /* ts->default_action_data. */
8672                 free(ts->default_action_data);
8673         }
8674
8675         free(p->table_state);
8676         p->table_state = NULL;
8677 }
8678
8679 static void
8680 table_state_free(struct rte_swx_pipeline *p)
8681 {
8682         table_state_build_free(p);
8683 }
8684
8685 /*
8686  * Register array.
8687  */
8688 static struct regarray *
8689 regarray_find(struct rte_swx_pipeline *p, const char *name)
8690 {
8691         struct regarray *elem;
8692
8693         TAILQ_FOREACH(elem, &p->regarrays, node)
8694                 if (!strcmp(elem->name, name))
8695                         return elem;
8696
8697         return NULL;
8698 }
8699
8700 static struct regarray *
8701 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8702 {
8703         struct regarray *elem = NULL;
8704
8705         TAILQ_FOREACH(elem, &p->regarrays, node)
8706                 if (elem->id == id)
8707                         return elem;
8708
8709         return NULL;
8710 }
8711
8712 int
8713 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8714                               const char *name,
8715                               uint32_t size,
8716                               uint64_t init_val)
8717 {
8718         struct regarray *r;
8719
8720         CHECK(p, EINVAL);
8721
8722         CHECK_NAME(name, EINVAL);
8723         CHECK(!regarray_find(p, name), EEXIST);
8724
8725         CHECK(size, EINVAL);
8726         size = rte_align32pow2(size);
8727
8728         /* Memory allocation. */
8729         r = calloc(1, sizeof(struct regarray));
8730         CHECK(r, ENOMEM);
8731
8732         /* Node initialization. */
8733         strcpy(r->name, name);
8734         r->init_val = init_val;
8735         r->size = size;
8736         r->id = p->n_regarrays;
8737
8738         /* Node add to tailq. */
8739         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8740         p->n_regarrays++;
8741
8742         return 0;
8743 }
8744
8745 static int
8746 regarray_build(struct rte_swx_pipeline *p)
8747 {
8748         struct regarray *regarray;
8749
8750         if (!p->n_regarrays)
8751                 return 0;
8752
8753         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8754         CHECK(p->regarray_runtime, ENOMEM);
8755
8756         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8757                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8758                 uint32_t i;
8759
8760                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8761                                          RTE_CACHE_LINE_SIZE,
8762                                          p->numa_node);
8763                 CHECK(r->regarray, ENOMEM);
8764
8765                 if (regarray->init_val)
8766                         for (i = 0; i < regarray->size; i++)
8767                                 r->regarray[i] = regarray->init_val;
8768
8769                 r->size_mask = regarray->size - 1;
8770         }
8771
8772         return 0;
8773 }
8774
8775 static void
8776 regarray_build_free(struct rte_swx_pipeline *p)
8777 {
8778         uint32_t i;
8779
8780         if (!p->regarray_runtime)
8781                 return;
8782
8783         for (i = 0; i < p->n_regarrays; i++) {
8784                 struct regarray *regarray = regarray_find_by_id(p, i);
8785                 struct regarray_runtime *r = &p->regarray_runtime[i];
8786
8787                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8788         }
8789
8790         free(p->regarray_runtime);
8791         p->regarray_runtime = NULL;
8792 }
8793
8794 static void
8795 regarray_free(struct rte_swx_pipeline *p)
8796 {
8797         regarray_build_free(p);
8798
8799         for ( ; ; ) {
8800                 struct regarray *elem;
8801
8802                 elem = TAILQ_FIRST(&p->regarrays);
8803                 if (!elem)
8804                         break;
8805
8806                 TAILQ_REMOVE(&p->regarrays, elem, node);
8807                 free(elem);
8808         }
8809 }
8810
8811 /*
8812  * Meter array.
8813  */
8814 static struct meter_profile *
8815 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8816 {
8817         struct meter_profile *elem;
8818
8819         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8820                 if (!strcmp(elem->name, name))
8821                         return elem;
8822
8823         return NULL;
8824 }
8825
8826 static struct metarray *
8827 metarray_find(struct rte_swx_pipeline *p, const char *name)
8828 {
8829         struct metarray *elem;
8830
8831         TAILQ_FOREACH(elem, &p->metarrays, node)
8832                 if (!strcmp(elem->name, name))
8833                         return elem;
8834
8835         return NULL;
8836 }
8837
8838 static struct metarray *
8839 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8840 {
8841         struct metarray *elem = NULL;
8842
8843         TAILQ_FOREACH(elem, &p->metarrays, node)
8844                 if (elem->id == id)
8845                         return elem;
8846
8847         return NULL;
8848 }
8849
8850 int
8851 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8852                                  const char *name,
8853                                  uint32_t size)
8854 {
8855         struct metarray *m;
8856
8857         CHECK(p, EINVAL);
8858
8859         CHECK_NAME(name, EINVAL);
8860         CHECK(!metarray_find(p, name), EEXIST);
8861
8862         CHECK(size, EINVAL);
8863         size = rte_align32pow2(size);
8864
8865         /* Memory allocation. */
8866         m = calloc(1, sizeof(struct metarray));
8867         CHECK(m, ENOMEM);
8868
8869         /* Node initialization. */
8870         strcpy(m->name, name);
8871         m->size = size;
8872         m->id = p->n_metarrays;
8873
8874         /* Node add to tailq. */
8875         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8876         p->n_metarrays++;
8877
8878         return 0;
8879 }
8880
8881 struct meter_profile meter_profile_default = {
8882         .node = {0},
8883         .name = "",
8884         .params = {0},
8885
8886         .profile = {
8887                 .cbs = 10000,
8888                 .pbs = 10000,
8889                 .cir_period = 1,
8890                 .cir_bytes_per_period = 1,
8891                 .pir_period = 1,
8892                 .pir_bytes_per_period = 1,
8893         },
8894
8895         .n_users = 0,
8896 };
8897
8898 static void
8899 meter_init(struct meter *m)
8900 {
8901         memset(m, 0, sizeof(struct meter));
8902         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8903         m->profile = &meter_profile_default;
8904         m->color_mask = RTE_COLOR_GREEN;
8905
8906         meter_profile_default.n_users++;
8907 }
8908
8909 static int
8910 metarray_build(struct rte_swx_pipeline *p)
8911 {
8912         struct metarray *m;
8913
8914         if (!p->n_metarrays)
8915                 return 0;
8916
8917         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8918         CHECK(p->metarray_runtime, ENOMEM);
8919
8920         TAILQ_FOREACH(m, &p->metarrays, node) {
8921                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8922                 uint32_t i;
8923
8924                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8925                                          RTE_CACHE_LINE_SIZE,
8926                                          p->numa_node);
8927                 CHECK(r->metarray, ENOMEM);
8928
8929                 for (i = 0; i < m->size; i++)
8930                         meter_init(&r->metarray[i]);
8931
8932                 r->size_mask = m->size - 1;
8933         }
8934
8935         return 0;
8936 }
8937
8938 static void
8939 metarray_build_free(struct rte_swx_pipeline *p)
8940 {
8941         uint32_t i;
8942
8943         if (!p->metarray_runtime)
8944                 return;
8945
8946         for (i = 0; i < p->n_metarrays; i++) {
8947                 struct metarray *m = metarray_find_by_id(p, i);
8948                 struct metarray_runtime *r = &p->metarray_runtime[i];
8949
8950                 env_free(r->metarray, m->size * sizeof(struct meter));
8951         }
8952
8953         free(p->metarray_runtime);
8954         p->metarray_runtime = NULL;
8955 }
8956
8957 static void
8958 metarray_free(struct rte_swx_pipeline *p)
8959 {
8960         metarray_build_free(p);
8961
8962         /* Meter arrays. */
8963         for ( ; ; ) {
8964                 struct metarray *elem;
8965
8966                 elem = TAILQ_FIRST(&p->metarrays);
8967                 if (!elem)
8968                         break;
8969
8970                 TAILQ_REMOVE(&p->metarrays, elem, node);
8971                 free(elem);
8972         }
8973
8974         /* Meter profiles. */
8975         for ( ; ; ) {
8976                 struct meter_profile *elem;
8977
8978                 elem = TAILQ_FIRST(&p->meter_profiles);
8979                 if (!elem)
8980                         break;
8981
8982                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8983                 free(elem);
8984         }
8985 }
8986
8987 /*
8988  * Pipeline.
8989  */
8990 void
8991 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8992 {
8993         void *lib;
8994
8995         if (!p)
8996                 return;
8997
8998         lib = p->lib;
8999
9000         free(p->instruction_data);
9001         free(p->instructions);
9002
9003         metarray_free(p);
9004         regarray_free(p);
9005         table_state_free(p);
9006         learner_free(p);
9007         selector_free(p);
9008         table_free(p);
9009         action_free(p);
9010         instruction_table_free(p);
9011         metadata_free(p);
9012         header_free(p);
9013         extern_func_free(p);
9014         extern_obj_free(p);
9015         port_out_free(p);
9016         port_in_free(p);
9017         struct_free(p);
9018
9019         free(p);
9020
9021         if (lib)
9022                 dlclose(lib);
9023 }
9024
9025 static int
9026 port_in_types_register(struct rte_swx_pipeline *p)
9027 {
9028         int status;
9029
9030         status = rte_swx_pipeline_port_in_type_register(p,
9031                 "ethdev",
9032                 &rte_swx_port_ethdev_reader_ops);
9033         if (status)
9034                 return status;
9035
9036         status = rte_swx_pipeline_port_in_type_register(p,
9037                 "ring",
9038                 &rte_swx_port_ring_reader_ops);
9039         if (status)
9040                 return status;
9041
9042 #ifdef RTE_PORT_PCAP
9043         status = rte_swx_pipeline_port_in_type_register(p,
9044                 "source",
9045                 &rte_swx_port_source_ops);
9046         if (status)
9047                 return status;
9048 #endif
9049
9050         status = rte_swx_pipeline_port_in_type_register(p,
9051                 "fd",
9052                 &rte_swx_port_fd_reader_ops);
9053         if (status)
9054                 return status;
9055
9056         return 0;
9057 }
9058
9059 static int
9060 port_out_types_register(struct rte_swx_pipeline *p)
9061 {
9062         int status;
9063
9064         status = rte_swx_pipeline_port_out_type_register(p,
9065                 "ethdev",
9066                 &rte_swx_port_ethdev_writer_ops);
9067         if (status)
9068                 return status;
9069
9070         status = rte_swx_pipeline_port_out_type_register(p,
9071                 "ring",
9072                 &rte_swx_port_ring_writer_ops);
9073         if (status)
9074                 return status;
9075
9076         status = rte_swx_pipeline_port_out_type_register(p,
9077                 "sink",
9078                 &rte_swx_port_sink_ops);
9079         if (status)
9080                 return status;
9081
9082         status = rte_swx_pipeline_port_out_type_register(p,
9083                 "fd",
9084                 &rte_swx_port_fd_writer_ops);
9085         if (status)
9086                 return status;
9087
9088         return 0;
9089 }
9090
9091 int
9092 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9093 {
9094         struct rte_swx_pipeline *pipeline = NULL;
9095         int status = 0;
9096
9097         /* Check input parameters. */
9098         CHECK(p, EINVAL);
9099
9100         /* Memory allocation. */
9101         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9102         if (!pipeline) {
9103                 status = -ENOMEM;
9104                 goto error;
9105         }
9106
9107         /* Initialization. */
9108         TAILQ_INIT(&pipeline->struct_types);
9109         TAILQ_INIT(&pipeline->port_in_types);
9110         TAILQ_INIT(&pipeline->ports_in);
9111         TAILQ_INIT(&pipeline->port_out_types);
9112         TAILQ_INIT(&pipeline->ports_out);
9113         TAILQ_INIT(&pipeline->extern_types);
9114         TAILQ_INIT(&pipeline->extern_objs);
9115         TAILQ_INIT(&pipeline->extern_funcs);
9116         TAILQ_INIT(&pipeline->headers);
9117         TAILQ_INIT(&pipeline->actions);
9118         TAILQ_INIT(&pipeline->table_types);
9119         TAILQ_INIT(&pipeline->tables);
9120         TAILQ_INIT(&pipeline->selectors);
9121         TAILQ_INIT(&pipeline->learners);
9122         TAILQ_INIT(&pipeline->regarrays);
9123         TAILQ_INIT(&pipeline->meter_profiles);
9124         TAILQ_INIT(&pipeline->metarrays);
9125
9126         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9127         pipeline->numa_node = numa_node;
9128
9129         status = port_in_types_register(pipeline);
9130         if (status)
9131                 goto error;
9132
9133         status = port_out_types_register(pipeline);
9134         if (status)
9135                 goto error;
9136
9137         *p = pipeline;
9138         return 0;
9139
9140 error:
9141         rte_swx_pipeline_free(pipeline);
9142         return status;
9143 }
9144
9145 int
9146 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9147                                      const char **instructions,
9148                                      uint32_t n_instructions)
9149 {
9150         int err;
9151         uint32_t i;
9152
9153         err = instruction_config(p, NULL, instructions, n_instructions);
9154         if (err)
9155                 return err;
9156
9157         /* Thread instruction pointer reset. */
9158         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9159                 struct thread *t = &p->threads[i];
9160
9161                 thread_ip_reset(p, t);
9162         }
9163
9164         return 0;
9165 }
9166
9167 static int
9168 pipeline_compile(struct rte_swx_pipeline *p);
9169
9170 int
9171 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9172 {
9173         int status;
9174
9175         CHECK(p, EINVAL);
9176         CHECK(p->build_done == 0, EEXIST);
9177
9178         status = port_in_build(p);
9179         if (status)
9180                 goto error;
9181
9182         status = port_out_build(p);
9183         if (status)
9184                 goto error;
9185
9186         status = struct_build(p);
9187         if (status)
9188                 goto error;
9189
9190         status = extern_obj_build(p);
9191         if (status)
9192                 goto error;
9193
9194         status = extern_func_build(p);
9195         if (status)
9196                 goto error;
9197
9198         status = header_build(p);
9199         if (status)
9200                 goto error;
9201
9202         status = metadata_build(p);
9203         if (status)
9204                 goto error;
9205
9206         status = instruction_table_build(p);
9207         if (status)
9208                 goto error;
9209
9210         status = action_build(p);
9211         if (status)
9212                 goto error;
9213
9214         status = table_build(p);
9215         if (status)
9216                 goto error;
9217
9218         status = selector_build(p);
9219         if (status)
9220                 goto error;
9221
9222         status = learner_build(p);
9223         if (status)
9224                 goto error;
9225
9226         status = table_state_build(p);
9227         if (status)
9228                 goto error;
9229
9230         status = regarray_build(p);
9231         if (status)
9232                 goto error;
9233
9234         status = metarray_build(p);
9235         if (status)
9236                 goto error;
9237
9238         p->build_done = 1;
9239
9240         pipeline_compile(p);
9241
9242         return 0;
9243
9244 error:
9245         metarray_build_free(p);
9246         regarray_build_free(p);
9247         table_state_build_free(p);
9248         learner_build_free(p);
9249         selector_build_free(p);
9250         table_build_free(p);
9251         action_build_free(p);
9252         instruction_table_build_free(p);
9253         metadata_build_free(p);
9254         header_build_free(p);
9255         extern_func_build_free(p);
9256         extern_obj_build_free(p);
9257         port_out_build_free(p);
9258         port_in_build_free(p);
9259         struct_build_free(p);
9260
9261         return status;
9262 }
9263
9264 void
9265 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9266 {
9267         uint32_t i;
9268
9269         for (i = 0; i < n_instructions; i++)
9270                 instr_exec(p);
9271 }
9272
9273 void
9274 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9275 {
9276         uint32_t i;
9277
9278         for (i = 0; i < p->n_ports_out; i++) {
9279                 struct port_out_runtime *port = &p->out[i];
9280
9281                 if (port->flush)
9282                         port->flush(port->obj);
9283         }
9284 }
9285
9286 /*
9287  * Control.
9288  */
9289 int
9290 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9291                               struct rte_swx_ctl_pipeline_info *pipeline)
9292 {
9293         struct action *action;
9294         struct table *table;
9295         uint32_t n_actions = 0, n_tables = 0;
9296
9297         if (!p || !pipeline)
9298                 return -EINVAL;
9299
9300         TAILQ_FOREACH(action, &p->actions, node)
9301                 n_actions++;
9302
9303         TAILQ_FOREACH(table, &p->tables, node)
9304                 n_tables++;
9305
9306         pipeline->n_ports_in = p->n_ports_in;
9307         pipeline->n_ports_out = p->n_ports_out;
9308         pipeline->n_actions = n_actions;
9309         pipeline->n_tables = n_tables;
9310         pipeline->n_selectors = p->n_selectors;
9311         pipeline->n_learners = p->n_learners;
9312         pipeline->n_regarrays = p->n_regarrays;
9313         pipeline->n_metarrays = p->n_metarrays;
9314
9315         return 0;
9316 }
9317
9318 int
9319 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9320 {
9321         if (!p || !numa_node)
9322                 return -EINVAL;
9323
9324         *numa_node = p->numa_node;
9325         return 0;
9326 }
9327
9328 int
9329 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9330                             uint32_t action_id,
9331                             struct rte_swx_ctl_action_info *action)
9332 {
9333         struct action *a = NULL;
9334
9335         if (!p || (action_id >= p->n_actions) || !action)
9336                 return -EINVAL;
9337
9338         a = action_find_by_id(p, action_id);
9339         if (!a)
9340                 return -EINVAL;
9341
9342         strcpy(action->name, a->name);
9343         action->n_args = a->st ? a->st->n_fields : 0;
9344         return 0;
9345 }
9346
9347 int
9348 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9349                                 uint32_t action_id,
9350                                 uint32_t action_arg_id,
9351                                 struct rte_swx_ctl_action_arg_info *action_arg)
9352 {
9353         struct action *a = NULL;
9354         struct field *arg = NULL;
9355
9356         if (!p || (action_id >= p->n_actions) || !action_arg)
9357                 return -EINVAL;
9358
9359         a = action_find_by_id(p, action_id);
9360         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9361                 return -EINVAL;
9362
9363         arg = &a->st->fields[action_arg_id];
9364         strcpy(action_arg->name, arg->name);
9365         action_arg->n_bits = arg->n_bits;
9366         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9367
9368         return 0;
9369 }
9370
9371 int
9372 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9373                            uint32_t table_id,
9374                            struct rte_swx_ctl_table_info *table)
9375 {
9376         struct table *t = NULL;
9377
9378         if (!p || !table)
9379                 return -EINVAL;
9380
9381         t = table_find_by_id(p, table_id);
9382         if (!t)
9383                 return -EINVAL;
9384
9385         strcpy(table->name, t->name);
9386         strcpy(table->args, t->args);
9387         table->n_match_fields = t->n_fields;
9388         table->n_actions = t->n_actions;
9389         table->default_action_is_const = t->default_action_is_const;
9390         table->size = t->size;
9391         return 0;
9392 }
9393
9394 int
9395 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9396         uint32_t table_id,
9397         uint32_t match_field_id,
9398         struct rte_swx_ctl_table_match_field_info *match_field)
9399 {
9400         struct table *t;
9401         struct match_field *f;
9402
9403         if (!p || (table_id >= p->n_tables) || !match_field)
9404                 return -EINVAL;
9405
9406         t = table_find_by_id(p, table_id);
9407         if (!t || (match_field_id >= t->n_fields))
9408                 return -EINVAL;
9409
9410         f = &t->fields[match_field_id];
9411         match_field->match_type = f->match_type;
9412         match_field->is_header = t->header ? 1 : 0;
9413         match_field->n_bits = f->field->n_bits;
9414         match_field->offset = f->field->offset;
9415
9416         return 0;
9417 }
9418
9419 int
9420 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9421         uint32_t table_id,
9422         uint32_t table_action_id,
9423         struct rte_swx_ctl_table_action_info *table_action)
9424 {
9425         struct table *t;
9426
9427         if (!p || (table_id >= p->n_tables) || !table_action)
9428                 return -EINVAL;
9429
9430         t = table_find_by_id(p, table_id);
9431         if (!t || (table_action_id >= t->n_actions))
9432                 return -EINVAL;
9433
9434         table_action->action_id = t->actions[table_action_id]->id;
9435
9436         table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9437         table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9438
9439         return 0;
9440 }
9441
9442 int
9443 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9444                           uint32_t table_id,
9445                           struct rte_swx_table_ops *table_ops,
9446                           int *is_stub)
9447 {
9448         struct table *t;
9449
9450         if (!p || (table_id >= p->n_tables))
9451                 return -EINVAL;
9452
9453         t = table_find_by_id(p, table_id);
9454         if (!t)
9455                 return -EINVAL;
9456
9457         if (t->type) {
9458                 if (table_ops)
9459                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9460                 *is_stub = 0;
9461         } else {
9462                 *is_stub = 1;
9463         }
9464
9465         return 0;
9466 }
9467
9468 int
9469 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9470                               uint32_t selector_id,
9471                               struct rte_swx_ctl_selector_info *selector)
9472 {
9473         struct selector *s = NULL;
9474
9475         if (!p || !selector)
9476                 return -EINVAL;
9477
9478         s = selector_find_by_id(p, selector_id);
9479         if (!s)
9480                 return -EINVAL;
9481
9482         strcpy(selector->name, s->name);
9483
9484         selector->n_selector_fields = s->n_selector_fields;
9485         selector->n_groups_max = s->n_groups_max;
9486         selector->n_members_per_group_max = s->n_members_per_group_max;
9487
9488         return 0;
9489 }
9490
9491 int
9492 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9493          uint32_t selector_id,
9494          struct rte_swx_ctl_table_match_field_info *field)
9495 {
9496         struct selector *s;
9497
9498         if (!p || (selector_id >= p->n_selectors) || !field)
9499                 return -EINVAL;
9500
9501         s = selector_find_by_id(p, selector_id);
9502         if (!s)
9503                 return -EINVAL;
9504
9505         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9506         field->is_header = 0;
9507         field->n_bits = s->group_id_field->n_bits;
9508         field->offset = s->group_id_field->offset;
9509
9510         return 0;
9511 }
9512
9513 int
9514 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9515          uint32_t selector_id,
9516          uint32_t selector_field_id,
9517          struct rte_swx_ctl_table_match_field_info *field)
9518 {
9519         struct selector *s;
9520         struct field *f;
9521
9522         if (!p || (selector_id >= p->n_selectors) || !field)
9523                 return -EINVAL;
9524
9525         s = selector_find_by_id(p, selector_id);
9526         if (!s || (selector_field_id >= s->n_selector_fields))
9527                 return -EINVAL;
9528
9529         f = s->selector_fields[selector_field_id];
9530         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9531         field->is_header = s->selector_header ? 1 : 0;
9532         field->n_bits = f->n_bits;
9533         field->offset = f->offset;
9534
9535         return 0;
9536 }
9537
9538 int
9539 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9540          uint32_t selector_id,
9541          struct rte_swx_ctl_table_match_field_info *field)
9542 {
9543         struct selector *s;
9544
9545         if (!p || (selector_id >= p->n_selectors) || !field)
9546                 return -EINVAL;
9547
9548         s = selector_find_by_id(p, selector_id);
9549         if (!s)
9550                 return -EINVAL;
9551
9552         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9553         field->is_header = 0;
9554         field->n_bits = s->member_id_field->n_bits;
9555         field->offset = s->member_id_field->offset;
9556
9557         return 0;
9558 }
9559
9560 int
9561 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9562                              uint32_t learner_id,
9563                              struct rte_swx_ctl_learner_info *learner)
9564 {
9565         struct learner *l = NULL;
9566
9567         if (!p || !learner)
9568                 return -EINVAL;
9569
9570         l = learner_find_by_id(p, learner_id);
9571         if (!l)
9572                 return -EINVAL;
9573
9574         strcpy(learner->name, l->name);
9575
9576         learner->n_match_fields = l->n_fields;
9577         learner->n_actions = l->n_actions;
9578         learner->default_action_is_const = l->default_action_is_const;
9579         learner->size = l->size;
9580
9581         return 0;
9582 }
9583
9584 int
9585 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9586                                          uint32_t learner_id,
9587                                          uint32_t match_field_id,
9588                                          struct rte_swx_ctl_table_match_field_info *match_field)
9589 {
9590         struct learner *l;
9591         struct field *f;
9592
9593         if (!p || (learner_id >= p->n_learners) || !match_field)
9594                 return -EINVAL;
9595
9596         l = learner_find_by_id(p, learner_id);
9597         if (!l || (match_field_id >= l->n_fields))
9598                 return -EINVAL;
9599
9600         f = l->fields[match_field_id];
9601         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9602         match_field->is_header = l->header ? 1 : 0;
9603         match_field->n_bits = f->n_bits;
9604         match_field->offset = f->offset;
9605
9606         return 0;
9607 }
9608
9609 int
9610 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9611                                     uint32_t learner_id,
9612                                     uint32_t learner_action_id,
9613                                     struct rte_swx_ctl_table_action_info *learner_action)
9614 {
9615         struct learner *l;
9616
9617         if (!p || (learner_id >= p->n_learners) || !learner_action)
9618                 return -EINVAL;
9619
9620         l = learner_find_by_id(p, learner_id);
9621         if (!l || (learner_action_id >= l->n_actions))
9622                 return -EINVAL;
9623
9624         learner_action->action_id = l->actions[learner_action_id]->id;
9625
9626         learner_action->action_is_for_table_entries =
9627                 l->action_is_for_table_entries[learner_action_id];
9628
9629         learner_action->action_is_for_default_entry =
9630                 l->action_is_for_default_entry[learner_action_id];
9631
9632         return 0;
9633 }
9634
9635 int
9636 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9637                                  struct rte_swx_table_state **table_state)
9638 {
9639         if (!p || !table_state || !p->build_done)
9640                 return -EINVAL;
9641
9642         *table_state = p->table_state;
9643         return 0;
9644 }
9645
9646 int
9647 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9648                                  struct rte_swx_table_state *table_state)
9649 {
9650         if (!p || !table_state || !p->build_done)
9651                 return -EINVAL;
9652
9653         p->table_state = table_state;
9654         return 0;
9655 }
9656
9657 int
9658 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9659                                         uint32_t port_id,
9660                                         struct rte_swx_port_in_stats *stats)
9661 {
9662         struct port_in *port;
9663
9664         if (!p || !stats)
9665                 return -EINVAL;
9666
9667         port = port_in_find(p, port_id);
9668         if (!port)
9669                 return -EINVAL;
9670
9671         port->type->ops.stats_read(port->obj, stats);
9672         return 0;
9673 }
9674
9675 int
9676 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9677                                          uint32_t port_id,
9678                                          struct rte_swx_port_out_stats *stats)
9679 {
9680         struct port_out *port;
9681
9682         if (!p || !stats)
9683                 return -EINVAL;
9684
9685         port = port_out_find(p, port_id);
9686         if (!port)
9687                 return -EINVAL;
9688
9689         port->type->ops.stats_read(port->obj, stats);
9690         return 0;
9691 }
9692
9693 int
9694 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9695                                       const char *table_name,
9696                                       struct rte_swx_table_stats *stats)
9697 {
9698         struct table *table;
9699         struct table_statistics *table_stats;
9700
9701         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9702                 return -EINVAL;
9703
9704         table = table_find(p, table_name);
9705         if (!table)
9706                 return -EINVAL;
9707
9708         table_stats = &p->table_stats[table->id];
9709
9710         memcpy(stats->n_pkts_action,
9711                table_stats->n_pkts_action,
9712                p->n_actions * sizeof(uint64_t));
9713
9714         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9715         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9716
9717         return 0;
9718 }
9719
9720 int
9721 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9722         const char *selector_name,
9723         struct rte_swx_pipeline_selector_stats *stats)
9724 {
9725         struct selector *s;
9726
9727         if (!p || !selector_name || !selector_name[0] || !stats)
9728                 return -EINVAL;
9729
9730         s = selector_find(p, selector_name);
9731         if (!s)
9732                 return -EINVAL;
9733
9734         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9735
9736         return 0;
9737 }
9738
9739 int
9740 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9741                                         const char *learner_name,
9742                                         struct rte_swx_learner_stats *stats)
9743 {
9744         struct learner *l;
9745         struct learner_statistics *learner_stats;
9746
9747         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9748                 return -EINVAL;
9749
9750         l = learner_find(p, learner_name);
9751         if (!l)
9752                 return -EINVAL;
9753
9754         learner_stats = &p->learner_stats[l->id];
9755
9756         memcpy(stats->n_pkts_action,
9757                learner_stats->n_pkts_action,
9758                p->n_actions * sizeof(uint64_t));
9759
9760         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9761         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9762
9763         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9764         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9765
9766         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9767
9768         return 0;
9769 }
9770
9771 int
9772 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9773                               uint32_t regarray_id,
9774                               struct rte_swx_ctl_regarray_info *regarray)
9775 {
9776         struct regarray *r;
9777
9778         if (!p || !regarray)
9779                 return -EINVAL;
9780
9781         r = regarray_find_by_id(p, regarray_id);
9782         if (!r)
9783                 return -EINVAL;
9784
9785         strcpy(regarray->name, r->name);
9786         regarray->size = r->size;
9787         return 0;
9788 }
9789
9790 int
9791 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9792                                    const char *regarray_name,
9793                                    uint32_t regarray_index,
9794                                    uint64_t *value)
9795 {
9796         struct regarray *regarray;
9797         struct regarray_runtime *r;
9798
9799         if (!p || !regarray_name || !value)
9800                 return -EINVAL;
9801
9802         regarray = regarray_find(p, regarray_name);
9803         if (!regarray || (regarray_index >= regarray->size))
9804                 return -EINVAL;
9805
9806         r = &p->regarray_runtime[regarray->id];
9807         *value = r->regarray[regarray_index];
9808         return 0;
9809 }
9810
9811 int
9812 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9813                                    const char *regarray_name,
9814                                    uint32_t regarray_index,
9815                                    uint64_t value)
9816 {
9817         struct regarray *regarray;
9818         struct regarray_runtime *r;
9819
9820         if (!p || !regarray_name)
9821                 return -EINVAL;
9822
9823         regarray = regarray_find(p, regarray_name);
9824         if (!regarray || (regarray_index >= regarray->size))
9825                 return -EINVAL;
9826
9827         r = &p->regarray_runtime[regarray->id];
9828         r->regarray[regarray_index] = value;
9829         return 0;
9830 }
9831
9832 int
9833 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9834                               uint32_t metarray_id,
9835                               struct rte_swx_ctl_metarray_info *metarray)
9836 {
9837         struct metarray *m;
9838
9839         if (!p || !metarray)
9840                 return -EINVAL;
9841
9842         m = metarray_find_by_id(p, metarray_id);
9843         if (!m)
9844                 return -EINVAL;
9845
9846         strcpy(metarray->name, m->name);
9847         metarray->size = m->size;
9848         return 0;
9849 }
9850
9851 int
9852 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9853                               const char *name,
9854                               struct rte_meter_trtcm_params *params)
9855 {
9856         struct meter_profile *mp;
9857         int status;
9858
9859         CHECK(p, EINVAL);
9860         CHECK_NAME(name, EINVAL);
9861         CHECK(params, EINVAL);
9862         CHECK(!meter_profile_find(p, name), EEXIST);
9863
9864         /* Node allocation. */
9865         mp = calloc(1, sizeof(struct meter_profile));
9866         CHECK(mp, ENOMEM);
9867
9868         /* Node initialization. */
9869         strcpy(mp->name, name);
9870         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9871         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9872         if (status) {
9873                 free(mp);
9874                 CHECK(0, EINVAL);
9875         }
9876
9877         /* Node add to tailq. */
9878         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9879
9880         return 0;
9881 }
9882
9883 int
9884 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9885                                  const char *name)
9886 {
9887         struct meter_profile *mp;
9888
9889         CHECK(p, EINVAL);
9890         CHECK_NAME(name, EINVAL);
9891
9892         mp = meter_profile_find(p, name);
9893         CHECK(mp, EINVAL);
9894         CHECK(!mp->n_users, EBUSY);
9895
9896         /* Remove node from tailq. */
9897         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9898         free(mp);
9899
9900         return 0;
9901 }
9902
9903 int
9904 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9905                         const char *metarray_name,
9906                         uint32_t metarray_index)
9907 {
9908         struct meter_profile *mp_old;
9909         struct metarray *metarray;
9910         struct metarray_runtime *metarray_runtime;
9911         struct meter *m;
9912
9913         CHECK(p, EINVAL);
9914         CHECK_NAME(metarray_name, EINVAL);
9915
9916         metarray = metarray_find(p, metarray_name);
9917         CHECK(metarray, EINVAL);
9918         CHECK(metarray_index < metarray->size, EINVAL);
9919
9920         metarray_runtime = &p->metarray_runtime[metarray->id];
9921         m = &metarray_runtime->metarray[metarray_index];
9922         mp_old = m->profile;
9923
9924         meter_init(m);
9925
9926         mp_old->n_users--;
9927
9928         return 0;
9929 }
9930
9931 int
9932 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9933                       const char *metarray_name,
9934                       uint32_t metarray_index,
9935                       const char *profile_name)
9936 {
9937         struct meter_profile *mp, *mp_old;
9938         struct metarray *metarray;
9939         struct metarray_runtime *metarray_runtime;
9940         struct meter *m;
9941
9942         CHECK(p, EINVAL);
9943         CHECK_NAME(metarray_name, EINVAL);
9944
9945         metarray = metarray_find(p, metarray_name);
9946         CHECK(metarray, EINVAL);
9947         CHECK(metarray_index < metarray->size, EINVAL);
9948
9949         mp = meter_profile_find(p, profile_name);
9950         CHECK(mp, EINVAL);
9951
9952         metarray_runtime = &p->metarray_runtime[metarray->id];
9953         m = &metarray_runtime->metarray[metarray_index];
9954         mp_old = m->profile;
9955
9956         memset(m, 0, sizeof(struct meter));
9957         rte_meter_trtcm_config(&m->m, &mp->profile);
9958         m->profile = mp;
9959         m->color_mask = RTE_COLORS;
9960
9961         mp->n_users++;
9962         mp_old->n_users--;
9963
9964         return 0;
9965 }
9966
9967 int
9968 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9969                              const char *metarray_name,
9970                              uint32_t metarray_index,
9971                              struct rte_swx_ctl_meter_stats *stats)
9972 {
9973         struct metarray *metarray;
9974         struct metarray_runtime *metarray_runtime;
9975         struct meter *m;
9976
9977         CHECK(p, EINVAL);
9978         CHECK_NAME(metarray_name, EINVAL);
9979
9980         metarray = metarray_find(p, metarray_name);
9981         CHECK(metarray, EINVAL);
9982         CHECK(metarray_index < metarray->size, EINVAL);
9983
9984         CHECK(stats, EINVAL);
9985
9986         metarray_runtime = &p->metarray_runtime[metarray->id];
9987         m = &metarray_runtime->metarray[metarray_index];
9988
9989         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9990         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9991
9992         return 0;
9993 }
9994
9995 /*
9996  * Pipeline compilation.
9997  */
9998 static const char *
9999 instr_type_to_name(struct instruction *instr)
10000 {
10001         switch (instr->type) {
10002         case INSTR_RX: return "INSTR_RX";
10003
10004         case INSTR_TX: return "INSTR_TX";
10005         case INSTR_TX_I: return "INSTR_TX_I";
10006         case INSTR_DROP: return "INSTR_DROP";
10007
10008         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10009         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10010         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10011         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10012         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10013         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10014         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10015         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10016
10017         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10018
10019         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10020
10021         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10022         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10023         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10024         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10025         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10026         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10027         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10028         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10029         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10030
10031         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10032         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10033
10034         case INSTR_MOV: return "INSTR_MOV";
10035         case INSTR_MOV_MH: return "INSTR_MOV_MH";
10036         case INSTR_MOV_HM: return "INSTR_MOV_HM";
10037         case INSTR_MOV_HH: return "INSTR_MOV_HH";
10038         case INSTR_MOV_I: return "INSTR_MOV_I";
10039
10040         case INSTR_DMA_HT: return "INSTR_DMA_HT";
10041         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10042         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10043         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10044         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10045         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10046         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10047         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10048
10049         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10050         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10051         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10052         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10053         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10054         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10055
10056         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10057         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10058         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10059         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10060         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10061         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10062
10063         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10064         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10065         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10066         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10067
10068         case INSTR_ALU_AND: return "INSTR_ALU_AND";
10069         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10070         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10071         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10072         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10073
10074         case INSTR_ALU_OR: return "INSTR_ALU_OR";
10075         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10076         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10077         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10078         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10079
10080         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10081         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10082         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10083         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10084         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10085
10086         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10087         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10088         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10089         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10090         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10091         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10092
10093         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10094         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10095         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10096         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10097         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10098         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10099
10100         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10101         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10102         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10103
10104         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10105         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10106         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10107         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10108         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10109         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10110
10111         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10112         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10113         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10114         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10115         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10116         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10117         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10118         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10119         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10120
10121         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10122         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10123         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10124         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10125         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10126         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10127         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10128         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10129         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10130
10131         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10132         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10133         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10134
10135         case INSTR_METER_HHM: return "INSTR_METER_HHM";
10136         case INSTR_METER_HHI: return "INSTR_METER_HHI";
10137         case INSTR_METER_HMM: return "INSTR_METER_HMM";
10138         case INSTR_METER_HMI: return "INSTR_METER_HMI";
10139         case INSTR_METER_MHM: return "INSTR_METER_MHM";
10140         case INSTR_METER_MHI: return "INSTR_METER_MHI";
10141         case INSTR_METER_MMM: return "INSTR_METER_MMM";
10142         case INSTR_METER_MMI: return "INSTR_METER_MMI";
10143         case INSTR_METER_IHM: return "INSTR_METER_IHM";
10144         case INSTR_METER_IHI: return "INSTR_METER_IHI";
10145         case INSTR_METER_IMM: return "INSTR_METER_IMM";
10146         case INSTR_METER_IMI: return "INSTR_METER_IMI";
10147
10148         case INSTR_TABLE: return "INSTR_TABLE";
10149         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10150         case INSTR_SELECTOR: return "INSTR_SELECTOR";
10151         case INSTR_LEARNER: return "INSTR_LEARNER";
10152         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10153
10154         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10155         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10156
10157         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10158         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10159
10160         case INSTR_JMP: return "INSTR_JMP";
10161         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10162         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10163         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10164         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10165         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10166         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10167         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10168         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10169         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10170         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10171         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10172         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10173         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10174         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10175         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10176         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10177         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10178         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10179         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10180         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10181         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10182         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10183         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10184         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10185         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10186         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10187         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10188         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10189
10190         case INSTR_RETURN: return "INSTR_RETURN";
10191
10192         default: return "INSTR_UNKNOWN";
10193         }
10194 }
10195
10196 typedef void
10197 (*instruction_export_t)(struct instruction *, FILE *);
10198
10199 static void
10200 instr_io_export(struct instruction *instr, FILE *f)
10201 {
10202         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10203
10204         /* n_io, n_io_imm, n_hdrs. */
10205         if (instr->type == INSTR_RX ||
10206             instr->type == INSTR_TX ||
10207             instr->type == INSTR_HDR_EXTRACT_M ||
10208             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10209                 n_io = 1;
10210
10211         if (instr->type == INSTR_TX_I)
10212                 n_io_imm = 1;
10213
10214         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10215                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10216
10217         if (instr->type == INSTR_HDR_EXTRACT_M ||
10218             instr->type == INSTR_HDR_LOOKAHEAD ||
10219             instr->type == INSTR_HDR_EMIT)
10220                 n_hdrs = 1;
10221
10222         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10223                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10224
10225         /* instr. */
10226         fprintf(f,
10227                 "\t{\n"
10228                 "\t\t.type = %s,\n",
10229                 instr_type_to_name(instr));
10230
10231         /* instr.io. */
10232         if (n_io || n_io_imm || n_hdrs)
10233                 fprintf(f,
10234                         "\t\t.io = {\n");
10235
10236         /* instr.io.io. */
10237         if (n_io)
10238                 fprintf(f,
10239                         "\t\t\t.io = {\n"
10240                         "\t\t\t\t.offset = %u,\n"
10241                         "\t\t\t\t.n_bits = %u,\n"
10242                         "\t\t\t},\n",
10243                         instr->io.io.offset,
10244                         instr->io.io.n_bits);
10245
10246         if (n_io_imm)
10247                 fprintf(f,
10248                         "\t\t\t.io = {\n"
10249                         "\t\t\t\t.val = %u,\n"
10250                         "\t\t\t},\n",
10251                         instr->io.io.val);
10252
10253         /* instr.io.hdr. */
10254         if (n_hdrs) {
10255                 fprintf(f,
10256                         "\t\t.hdr = {\n");
10257
10258                 /* instr.io.hdr.header_id. */
10259                 fprintf(f,
10260                         "\t\t\t.header_id = {");
10261
10262                 for (i = 0; i < n_hdrs; i++)
10263                         fprintf(f,
10264                                 "%u, ",
10265                                 instr->io.hdr.header_id[i]);
10266
10267                 fprintf(f,
10268                         "},\n");
10269
10270                 /* instr.io.hdr.struct_id. */
10271                 fprintf(f,
10272                         "\t\t\t.struct_id = {");
10273
10274                 for (i = 0; i < n_hdrs; i++)
10275                         fprintf(f,
10276                                 "%u, ",
10277                                 instr->io.hdr.struct_id[i]);
10278
10279                 fprintf(f,
10280                         "},\n");
10281
10282                 /* instr.io.hdr.n_bytes. */
10283                 fprintf(f,
10284                         "\t\t\t.n_bytes = {");
10285
10286                 for (i = 0; i < n_hdrs; i++)
10287                         fprintf(f,
10288                                 "%u, ",
10289                                 instr->io.hdr.n_bytes[i]);
10290
10291                 fprintf(f,
10292                         "},\n");
10293
10294                 /* instr.io.hdr - closing curly brace. */
10295                 fprintf(f,
10296                         "\t\t\t}\n,");
10297         }
10298
10299         /* instr.io - closing curly brace. */
10300         if (n_io || n_io_imm || n_hdrs)
10301                 fprintf(f,
10302                         "\t\t},\n");
10303
10304         /* instr - closing curly brace. */
10305         fprintf(f,
10306                 "\t},\n");
10307 }
10308
10309 static void
10310 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10311 {
10312         fprintf(f,
10313                 "\t{\n"
10314                 "\t\t.type = %s,\n"
10315                 "\t\t.valid = {\n"
10316                 "\t\t\t.header_id = %u,\n"
10317                 "\t\t},\n"
10318                 "\t},\n",
10319                 instr_type_to_name(instr),
10320                 instr->valid.header_id);
10321 }
10322
10323 static void
10324 instr_mov_export(struct instruction *instr, FILE *f)
10325 {
10326         if (instr->type != INSTR_MOV_I)
10327                 fprintf(f,
10328                         "\t{\n"
10329                         "\t\t.type = %s,\n"
10330                         "\t\t.mov = {\n"
10331                         "\t\t\t.dst = {\n"
10332                         "\t\t\t\t.struct_id = %u,\n"
10333                         "\t\t\t\t.n_bits = %u,\n"
10334                         "\t\t\t\t.offset = %u,\n"
10335                         "\t\t\t},\n"
10336                         "\t\t\t.src = {\n"
10337                         "\t\t\t\t.struct_id = %u,\n"
10338                         "\t\t\t\t.n_bits = %u,\n"
10339                         "\t\t\t\t.offset = %u,\n"
10340                         "\t\t\t},\n"
10341                         "\t\t},\n"
10342                         "\t},\n",
10343                         instr_type_to_name(instr),
10344                         instr->mov.dst.struct_id,
10345                         instr->mov.dst.n_bits,
10346                         instr->mov.dst.offset,
10347                         instr->mov.src.struct_id,
10348                         instr->mov.src.n_bits,
10349                         instr->mov.src.offset);
10350         else
10351                 fprintf(f,
10352                         "\t{\n"
10353                         "\t\t.type = %s,\n"
10354                         "\t\t.mov = {\n"
10355                         "\t\t\t.dst = {\n"
10356                         "\t\t\t\t.struct_id = %u,\n"
10357                         "\t\t\t\t.n_bits = %u,\n"
10358                         "\t\t\t\t.offset = %u,\n"
10359                         "\t\t\t}\n,"
10360                         "\t\t\t.src_val = %" PRIu64 ",\n"
10361                         "\t\t},\n"
10362                         "\t},\n",
10363                         instr_type_to_name(instr),
10364                         instr->mov.dst.struct_id,
10365                         instr->mov.dst.n_bits,
10366                         instr->mov.dst.offset,
10367                         instr->mov.src_val);
10368 }
10369
10370 static void
10371 instr_dma_ht_export(struct instruction *instr, FILE *f)
10372 {
10373         uint32_t n_dma = 0, i;
10374
10375         /* n_dma. */
10376         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10377
10378         /* instr. */
10379         fprintf(f,
10380                 "\t{\n"
10381                 "\t\t.type = %s,\n",
10382                 instr_type_to_name(instr));
10383
10384         /* instr.dma. */
10385         fprintf(f,
10386                 "\t\t.dma = {\n");
10387
10388         /* instr.dma.dst. */
10389         fprintf(f,
10390                 "\t\t\t.dst = {\n");
10391
10392         /* instr.dma.dst.header_id. */
10393         fprintf(f,
10394                 "\t\t\t\t.header_id = {");
10395
10396         for (i = 0; i < n_dma; i++)
10397                 fprintf(f,
10398                         "%u, ",
10399                         instr->dma.dst.header_id[i]);
10400
10401         fprintf(f,
10402                 "},\n");
10403
10404         /* instr.dma.dst.struct_id. */
10405         fprintf(f,
10406                 "\t\t\t\t.struct_id = {");
10407
10408         for (i = 0; i < n_dma; i++)
10409                 fprintf(f,
10410                         "%u, ",
10411                         instr->dma.dst.struct_id[i]);
10412
10413         fprintf(f,
10414                 "},\n");
10415
10416         /* instr.dma.dst - closing curly brace. */
10417         fprintf(f,
10418                 "\t\t\t},\n");
10419
10420         /* instr.dma.src. */
10421         fprintf(f,
10422                 "\t\t\t.src = {\n");
10423
10424         /* instr.dma.src.offset. */
10425         fprintf(f,
10426                 "\t\t\t\t.offset = {");
10427
10428         for (i = 0; i < n_dma; i++)
10429                 fprintf(f,
10430                         "%u, ",
10431                         instr->dma.src.offset[i]);
10432
10433         fprintf(f,
10434                 "},\n");
10435
10436         /* instr.dma.src - closing curly brace. */
10437         fprintf(f,
10438                 "\t\t\t},\n");
10439
10440         /* instr.dma.n_bytes. */
10441         fprintf(f,
10442                 "\t\t\t.n_bytes = {");
10443
10444         for (i = 0; i < n_dma; i++)
10445                 fprintf(f,
10446                         "%u, ",
10447                         instr->dma.n_bytes[i]);
10448
10449         fprintf(f,
10450                 "},\n");
10451
10452         /* instr.dma - closing curly brace. */
10453         fprintf(f,
10454                 "\t\t},\n");
10455
10456         /* instr - closing curly brace. */
10457         fprintf(f,
10458                 "\t},\n");
10459 }
10460
10461 static void
10462 instr_alu_export(struct instruction *instr, FILE *f)
10463 {
10464         int imm = 0;
10465
10466         if (instr->type == INSTR_ALU_ADD_MI ||
10467             instr->type == INSTR_ALU_ADD_HI ||
10468             instr->type == INSTR_ALU_SUB_MI ||
10469             instr->type == INSTR_ALU_SUB_HI ||
10470             instr->type == INSTR_ALU_SHL_MI ||
10471             instr->type == INSTR_ALU_SHL_HI ||
10472             instr->type == INSTR_ALU_SHR_MI ||
10473             instr->type == INSTR_ALU_SHR_HI ||
10474             instr->type == INSTR_ALU_AND_I ||
10475             instr->type == INSTR_ALU_OR_I ||
10476             instr->type == INSTR_ALU_XOR_I)
10477                 imm = 1;
10478
10479         if (!imm)
10480                 fprintf(f,
10481                         "\t{\n"
10482                         "\t\t.type = %s,\n"
10483                         "\t\t.alu = {\n"
10484                         "\t\t\t.dst = {\n"
10485                         "\t\t\t\t.struct_id = %u,\n"
10486                         "\t\t\t\t.n_bits = %u,\n"
10487                         "\t\t\t\t.offset = %u,\n"
10488                         "\t\t\t},\n"
10489                         "\t\t\t.src = {\n"
10490                         "\t\t\t\t.struct_id = %u,\n"
10491                         "\t\t\t\t.n_bits = %u,\n"
10492                         "\t\t\t\t.offset = %u,\n"
10493                         "\t\t\t},\n"
10494                         "\t\t},\n"
10495                         "\t},\n",
10496                         instr_type_to_name(instr),
10497                         instr->alu.dst.struct_id,
10498                         instr->alu.dst.n_bits,
10499                         instr->alu.dst.offset,
10500                         instr->alu.src.struct_id,
10501                         instr->alu.src.n_bits,
10502                         instr->alu.src.offset);
10503         else
10504                 fprintf(f,
10505                         "\t{\n"
10506                         "\t\t.type = %s,\n"
10507                         "\t\t.alu = {\n"
10508                         "\t\t\t.dst = {\n"
10509                         "\t\t\t\t.struct_id = %u,\n"
10510                         "\t\t\t\t.n_bits = %u,\n"
10511                         "\t\t\t\t.offset = %u,\n"
10512                         "\t\t\t}\n,"
10513                         "\t\t\t.src_val = %" PRIu64 ",\n"
10514                         "\t\t},\n"
10515                         "\t},\n",
10516                         instr_type_to_name(instr),
10517                         instr->alu.dst.struct_id,
10518                         instr->alu.dst.n_bits,
10519                         instr->alu.dst.offset,
10520                         instr->alu.src_val);
10521 }
10522
10523 static void
10524 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10525 {
10526         int prefetch  = 0, idx_imm = 0, src_imm = 0;
10527
10528         if (instr->type == INSTR_REGPREFETCH_RH ||
10529             instr->type == INSTR_REGPREFETCH_RM ||
10530             instr->type == INSTR_REGPREFETCH_RI)
10531                 prefetch = 1;
10532
10533         /* index is the 3rd operand for the regrd instruction and the 2nd
10534          * operand for the regwr and regadd instructions.
10535          */
10536         if (instr->type == INSTR_REGPREFETCH_RI ||
10537             instr->type == INSTR_REGRD_HRI ||
10538             instr->type == INSTR_REGRD_MRI ||
10539             instr->type == INSTR_REGWR_RIH ||
10540             instr->type == INSTR_REGWR_RIM ||
10541             instr->type == INSTR_REGWR_RII ||
10542             instr->type == INSTR_REGADD_RIH ||
10543             instr->type == INSTR_REGADD_RIM ||
10544             instr->type == INSTR_REGADD_RII)
10545                 idx_imm = 1;
10546
10547         /* src is the 3rd operand for the regwr and regadd instructions. */
10548         if (instr->type == INSTR_REGWR_RHI ||
10549             instr->type == INSTR_REGWR_RMI ||
10550             instr->type == INSTR_REGWR_RII ||
10551             instr->type == INSTR_REGADD_RHI ||
10552             instr->type == INSTR_REGADD_RMI ||
10553             instr->type == INSTR_REGADD_RII)
10554                 src_imm = 1;
10555
10556         /* instr.regarray.regarray_id. */
10557         fprintf(f,
10558                 "\t{\n"
10559                 "\t\t.type = %s,\n"
10560                 "\t\t.regarray = {\n"
10561                 "\t\t\t.regarray_id = %u,\n",
10562                 instr_type_to_name(instr),
10563                 instr->regarray.regarray_id);
10564
10565         /* instr.regarray.idx / instr.regarray.idx_val. */
10566         if (!idx_imm)
10567                 fprintf(f,
10568                         "\t\t\t\t.idx = {\n"
10569                         "\t\t\t\t\t.struct_id = %u,\n"
10570                         "\t\t\t\t\t.n_bits = %u,\n"
10571                         "\t\t\t\t\t.offset = %u,\n"
10572                         "\t\t\t\t},\n",
10573                         instr->regarray.idx.struct_id,
10574                         instr->regarray.idx.n_bits,
10575                         instr->regarray.idx.offset);
10576         else
10577                 fprintf(f,
10578                         "\t\t\t\t.idx_val = %u,\n",
10579                         instr->regarray.idx_val);
10580
10581         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10582         if (!prefetch) {
10583                 if (!src_imm)
10584                         fprintf(f,
10585                                 "\t\t\t\t.dstsrc = {\n"
10586                                 "\t\t\t\t\t.struct_id = %u,\n"
10587                                 "\t\t\t\t\t.n_bits = %u,\n"
10588                                 "\t\t\t\t\t.offset = %u,\n"
10589                                 "\t\t\t\t},\n",
10590                                 instr->regarray.dstsrc.struct_id,
10591                                 instr->regarray.dstsrc.n_bits,
10592                                 instr->regarray.dstsrc.offset);
10593                 else
10594                         fprintf(f,
10595                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10596                                 instr->regarray.dstsrc_val);
10597         }
10598
10599         /* instr.regarray and instr - closing curly braces. */
10600         fprintf(f,
10601                 "\t\t},\n"
10602                 "\t},\n");
10603 }
10604
10605 static void
10606 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10607 {
10608         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
10609
10610         if (instr->type == INSTR_METPREFETCH_H ||
10611             instr->type == INSTR_METPREFETCH_M ||
10612             instr->type == INSTR_METPREFETCH_I)
10613                 prefetch = 1;
10614
10615         /* idx_imm. */
10616         if (instr->type == INSTR_METPREFETCH_I ||
10617             instr->type == INSTR_METER_IHM ||
10618             instr->type == INSTR_METER_IHI ||
10619             instr->type == INSTR_METER_IMM ||
10620             instr->type == INSTR_METER_IMI)
10621                 idx_imm = 1;
10622
10623         /* color_in_imm. */
10624         if (instr->type == INSTR_METER_HHI ||
10625             instr->type == INSTR_METER_HMI ||
10626             instr->type == INSTR_METER_MHI ||
10627             instr->type == INSTR_METER_MMI ||
10628             instr->type == INSTR_METER_IHI ||
10629             instr->type == INSTR_METER_IMI)
10630                 color_in_imm = 1;
10631
10632         /* instr.meter.metarray_id. */
10633         fprintf(f,
10634                 "\t{\n"
10635                 "\t\t.type = %s,\n"
10636                 "\t\t.meter = {\n"
10637                 "\t\t\t.metarray_id = %u,\n",
10638                 instr_type_to_name(instr),
10639                 instr->meter.metarray_id);
10640
10641         /* instr.meter.idx / instr.meter.idx_val. */
10642         if (!idx_imm)
10643                 fprintf(f,
10644                         "\t\t\t.idx = {\n"
10645                         "\t\t\t\t.struct_id = %u,\n"
10646                         "\t\t\t\t.n_bits = %u,\n"
10647                         "\t\t\t\t.offset = %u,\n"
10648                         "\t\t\t},\n",
10649                         instr->meter.idx.struct_id,
10650                         instr->meter.idx.n_bits,
10651                         instr->meter.idx.offset);
10652         else
10653                 fprintf(f,
10654                         "\t\t\t.idx_val = %u,\n",
10655                         instr->meter.idx_val);
10656
10657         if (!prefetch) {
10658                 /* instr.meter.length. */
10659                 fprintf(f,
10660                         "\t\t\t.length = {\n"
10661                         "\t\t\t\t.struct_id = %u,\n"
10662                         "\t\t\t\t.n_bits = %u,\n"
10663                         "\t\t\t\t.offset = %u,\n"
10664                         "\t\t\t},\n",
10665                         instr->meter.length.struct_id,
10666                         instr->meter.length.n_bits,
10667                         instr->meter.length.offset);
10668
10669                 /* instr.meter.color_in / instr.meter.color_in_val. */
10670                 if (!color_in_imm)
10671                         fprintf(f,
10672                                 "\t\t\t.color_in = {\n"
10673                                 "\t\t\t\t.struct_id = %u,\n"
10674                                 "\t\t\t\t.n_bits = %u,\n"
10675                                 "\t\t\t\t.offset = %u,\n"
10676                                 "\t\t\t},\n",
10677                                 instr->meter.color_in.struct_id,
10678                                 instr->meter.color_in.n_bits,
10679                                 instr->meter.color_in.offset);
10680                 else
10681                         fprintf(f,
10682                                 "\t\t\t.color_in_val = %u,\n",
10683                                 (uint32_t)instr->meter.color_in_val);
10684
10685                 /* instr.meter.color_out. */
10686                 fprintf(f,
10687                         "\t\t\t.color_out = {\n"
10688                         "\t\t\t\t.struct_id = %u,\n"
10689                         "\t\t\t\t.n_bits = %u,\n"
10690                         "\t\t\t\t.offset = %u,\n"
10691                         "\t\t\t},\n",
10692                         instr->meter.color_out.struct_id,
10693                         instr->meter.color_out.n_bits,
10694                         instr->meter.color_out.offset);
10695         }
10696
10697         /* instr.meter and instr - closing curly braces. */
10698         fprintf(f,
10699                 "\t\t},\n"
10700                 "\t},\n");
10701 }
10702
10703 static void
10704 instr_table_export(struct instruction *instr,
10705                 FILE *f)
10706 {
10707         fprintf(f,
10708                 "\t{\n"
10709                 "\t\t.type = %s,\n"
10710                 "\t\t.table = {\n"
10711                 "\t\t\t.table_id = %u,\n"
10712                 "\t\t},\n"
10713                 "\t},\n",
10714                 instr_type_to_name(instr),
10715                 instr->table.table_id);
10716 }
10717
10718 static void
10719 instr_learn_export(struct instruction *instr, FILE *f)
10720 {
10721         fprintf(f,
10722                 "\t{\n"
10723                 "\t\t.type = %s,\n"
10724                 "\t\t.learn = {\n"
10725                 "\t\t\t\t.action_id = %u,\n"
10726                 "\t\t},\n"
10727                 "\t},\n",
10728                 instr_type_to_name(instr),
10729                 instr->learn.action_id);
10730 }
10731
10732 static void
10733 instr_forget_export(struct instruction *instr, FILE *f)
10734 {
10735         fprintf(f,
10736                 "\t{\n"
10737                 "\t\t.type = %s,\n"
10738                 "\t},\n",
10739                 instr_type_to_name(instr));
10740 }
10741
10742 static void
10743 instr_extern_export(struct instruction *instr, FILE *f)
10744 {
10745         if (instr->type == INSTR_EXTERN_OBJ)
10746                 fprintf(f,
10747                         "\t{\n"
10748                         "\t\t.type = %s,\n"
10749                         "\t\t.ext_obj = {\n"
10750                         "\t\t\t.ext_obj_id = %u,\n"
10751                         "\t\t\t.func_id = %u,\n"
10752                         "\t\t},\n"
10753                         "\t},\n",
10754                         instr_type_to_name(instr),
10755                         instr->ext_obj.ext_obj_id,
10756                         instr->ext_obj.func_id);
10757         else
10758                 fprintf(f,
10759                         "\t{\n"
10760                         "\t\t.type = %s,\n"
10761                         "\t\t.ext_func = {\n"
10762                         "\t\t\t.ext_func_id = %u,\n"
10763                         "\t\t},\n"
10764                         "\t},\n",
10765                         instr_type_to_name(instr),
10766                         instr->ext_func.ext_func_id);
10767 }
10768
10769 static void
10770 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10771 {
10772         fprintf(f,
10773                 "\t{\n"
10774                 "\t\t.type = %s,\n"
10775                 "\t\t.jmp = {\n"
10776                 "\t\t\t.ip = NULL,\n",
10777                 instr_type_to_name(instr));
10778
10779         switch (instr->type) {
10780         case INSTR_JMP_VALID:
10781         case INSTR_JMP_INVALID:
10782                 fprintf(f,
10783                         "\t\t\t.header_id = %u,\n",
10784                         instr->jmp.header_id);
10785                 break;
10786
10787         case INSTR_JMP_ACTION_HIT:
10788         case INSTR_JMP_ACTION_MISS:
10789                 fprintf(f,
10790                         "\t\t\t.action_id = %u,\n",
10791                         instr->jmp.action_id);
10792                 break;
10793
10794         case INSTR_JMP_EQ:
10795         case INSTR_JMP_EQ_MH:
10796         case INSTR_JMP_EQ_HM:
10797         case INSTR_JMP_EQ_HH:
10798         case INSTR_JMP_NEQ:
10799         case INSTR_JMP_NEQ_MH:
10800         case INSTR_JMP_NEQ_HM:
10801         case INSTR_JMP_NEQ_HH:
10802         case INSTR_JMP_LT:
10803         case INSTR_JMP_LT_MH:
10804         case INSTR_JMP_LT_HM:
10805         case INSTR_JMP_LT_HH:
10806         case INSTR_JMP_GT:
10807         case INSTR_JMP_GT_MH:
10808         case INSTR_JMP_GT_HM:
10809         case INSTR_JMP_GT_HH:
10810                 fprintf(f,
10811                         "\t\t\t.a = {\n"
10812                         "\t\t\t\t.struct_id = %u,\n"
10813                         "\t\t\t\t.n_bits = %u,\n"
10814                         "\t\t\t\t.offset = %u,\n"
10815                         "\t\t\t},\n"
10816                         "\t\t\t.b = {\n"
10817                         "\t\t\t\t.struct_id = %u,\n"
10818                         "\t\t\t\t.n_bits = %u,\n"
10819                         "\t\t\t\t.offset = %u,\n"
10820                         "\t\t\t},\n",
10821                         instr->jmp.a.struct_id,
10822                         instr->jmp.a.n_bits,
10823                         instr->jmp.a.offset,
10824                         instr->jmp.b.struct_id,
10825                         instr->jmp.b.n_bits,
10826                         instr->jmp.b.offset);
10827                 break;
10828
10829         case INSTR_JMP_EQ_I:
10830         case INSTR_JMP_NEQ_I:
10831         case INSTR_JMP_LT_MI:
10832         case INSTR_JMP_LT_HI:
10833         case INSTR_JMP_GT_MI:
10834         case INSTR_JMP_GT_HI:
10835                 fprintf(f,
10836                         "\t\t\t.a = {\n"
10837                         "\t\t\t\t.struct_id = %u,\n"
10838                         "\t\t\t\t.n_bits = %u,\n"
10839                         "\t\t\t\t.offset = %u,\n"
10840                         "\t\t\t}\n,"
10841                         "\t\t\t.b_val = %" PRIu64 ",\n",
10842                         instr->jmp.a.struct_id,
10843                         instr->jmp.a.n_bits,
10844                         instr->jmp.a.offset,
10845                         instr->jmp.b_val);
10846                 break;
10847
10848         default:
10849                 break;
10850         }
10851
10852         fprintf(f,
10853                 "\t\t},\n"
10854                 "\t},\n");
10855 }
10856
10857 static void
10858 instr_return_export(struct instruction *instr,
10859                 FILE *f)
10860 {
10861         fprintf(f,
10862                 "\t{\n"
10863                 "\t\t.type = %s,\n",
10864                 instr_type_to_name(instr));
10865
10866         fprintf(f,
10867                 "\t},\n");
10868 }
10869
10870 static instruction_export_t export_table[] = {
10871         [INSTR_RX] = instr_io_export,
10872
10873         [INSTR_TX] = instr_io_export,
10874         [INSTR_TX_I] = instr_io_export,
10875         [INSTR_DROP] = instr_io_export,
10876
10877         [INSTR_HDR_EXTRACT] = instr_io_export,
10878         [INSTR_HDR_EXTRACT2] = instr_io_export,
10879         [INSTR_HDR_EXTRACT3] = instr_io_export,
10880         [INSTR_HDR_EXTRACT4] = instr_io_export,
10881         [INSTR_HDR_EXTRACT5] = instr_io_export,
10882         [INSTR_HDR_EXTRACT6] = instr_io_export,
10883         [INSTR_HDR_EXTRACT7] = instr_io_export,
10884         [INSTR_HDR_EXTRACT8] = instr_io_export,
10885
10886         [INSTR_HDR_EXTRACT_M] = instr_io_export,
10887
10888         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10889
10890         [INSTR_HDR_EMIT] = instr_io_export,
10891         [INSTR_HDR_EMIT_TX] = instr_io_export,
10892         [INSTR_HDR_EMIT2_TX] = instr_io_export,
10893         [INSTR_HDR_EMIT3_TX] = instr_io_export,
10894         [INSTR_HDR_EMIT4_TX] = instr_io_export,
10895         [INSTR_HDR_EMIT5_TX] = instr_io_export,
10896         [INSTR_HDR_EMIT6_TX] = instr_io_export,
10897         [INSTR_HDR_EMIT7_TX] = instr_io_export,
10898         [INSTR_HDR_EMIT8_TX] = instr_io_export,
10899
10900         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10901         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10902
10903         [INSTR_MOV] = instr_mov_export,
10904         [INSTR_MOV_MH] = instr_mov_export,
10905         [INSTR_MOV_HM] = instr_mov_export,
10906         [INSTR_MOV_HH] = instr_mov_export,
10907         [INSTR_MOV_I] = instr_mov_export,
10908
10909         [INSTR_DMA_HT]  = instr_dma_ht_export,
10910         [INSTR_DMA_HT2] = instr_dma_ht_export,
10911         [INSTR_DMA_HT3] = instr_dma_ht_export,
10912         [INSTR_DMA_HT4] = instr_dma_ht_export,
10913         [INSTR_DMA_HT5] = instr_dma_ht_export,
10914         [INSTR_DMA_HT6] = instr_dma_ht_export,
10915         [INSTR_DMA_HT7] = instr_dma_ht_export,
10916         [INSTR_DMA_HT8] = instr_dma_ht_export,
10917
10918         [INSTR_ALU_ADD] = instr_alu_export,
10919         [INSTR_ALU_ADD_MH] = instr_alu_export,
10920         [INSTR_ALU_ADD_HM] = instr_alu_export,
10921         [INSTR_ALU_ADD_HH] = instr_alu_export,
10922         [INSTR_ALU_ADD_MI] = instr_alu_export,
10923         [INSTR_ALU_ADD_HI] = instr_alu_export,
10924
10925         [INSTR_ALU_SUB] = instr_alu_export,
10926         [INSTR_ALU_SUB_MH] = instr_alu_export,
10927         [INSTR_ALU_SUB_HM] = instr_alu_export,
10928         [INSTR_ALU_SUB_HH] = instr_alu_export,
10929         [INSTR_ALU_SUB_MI] = instr_alu_export,
10930         [INSTR_ALU_SUB_HI] = instr_alu_export,
10931
10932         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10933         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10934         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10935         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10936
10937         [INSTR_ALU_AND] = instr_alu_export,
10938         [INSTR_ALU_AND_MH] = instr_alu_export,
10939         [INSTR_ALU_AND_HM] = instr_alu_export,
10940         [INSTR_ALU_AND_HH] = instr_alu_export,
10941         [INSTR_ALU_AND_I] = instr_alu_export,
10942
10943         [INSTR_ALU_OR] = instr_alu_export,
10944         [INSTR_ALU_OR_MH] = instr_alu_export,
10945         [INSTR_ALU_OR_HM] = instr_alu_export,
10946         [INSTR_ALU_OR_HH] = instr_alu_export,
10947         [INSTR_ALU_OR_I] = instr_alu_export,
10948
10949         [INSTR_ALU_XOR] = instr_alu_export,
10950         [INSTR_ALU_XOR_MH] = instr_alu_export,
10951         [INSTR_ALU_XOR_HM] = instr_alu_export,
10952         [INSTR_ALU_XOR_HH] = instr_alu_export,
10953         [INSTR_ALU_XOR_I] = instr_alu_export,
10954
10955         [INSTR_ALU_SHL] = instr_alu_export,
10956         [INSTR_ALU_SHL_MH] = instr_alu_export,
10957         [INSTR_ALU_SHL_HM] = instr_alu_export,
10958         [INSTR_ALU_SHL_HH] = instr_alu_export,
10959         [INSTR_ALU_SHL_MI] = instr_alu_export,
10960         [INSTR_ALU_SHL_HI] = instr_alu_export,
10961
10962         [INSTR_ALU_SHR] = instr_alu_export,
10963         [INSTR_ALU_SHR_MH] = instr_alu_export,
10964         [INSTR_ALU_SHR_HM] = instr_alu_export,
10965         [INSTR_ALU_SHR_HH] = instr_alu_export,
10966         [INSTR_ALU_SHR_MI] = instr_alu_export,
10967         [INSTR_ALU_SHR_HI] = instr_alu_export,
10968
10969         [INSTR_REGPREFETCH_RH] = instr_reg_export,
10970         [INSTR_REGPREFETCH_RM] = instr_reg_export,
10971         [INSTR_REGPREFETCH_RI] = instr_reg_export,
10972
10973         [INSTR_REGRD_HRH] = instr_reg_export,
10974         [INSTR_REGRD_HRM] = instr_reg_export,
10975         [INSTR_REGRD_MRH] = instr_reg_export,
10976         [INSTR_REGRD_MRM] = instr_reg_export,
10977         [INSTR_REGRD_HRI] = instr_reg_export,
10978         [INSTR_REGRD_MRI] = instr_reg_export,
10979
10980         [INSTR_REGWR_RHH] = instr_reg_export,
10981         [INSTR_REGWR_RHM] = instr_reg_export,
10982         [INSTR_REGWR_RMH] = instr_reg_export,
10983         [INSTR_REGWR_RMM] = instr_reg_export,
10984         [INSTR_REGWR_RHI] = instr_reg_export,
10985         [INSTR_REGWR_RMI] = instr_reg_export,
10986         [INSTR_REGWR_RIH] = instr_reg_export,
10987         [INSTR_REGWR_RIM] = instr_reg_export,
10988         [INSTR_REGWR_RII] = instr_reg_export,
10989
10990         [INSTR_REGADD_RHH] = instr_reg_export,
10991         [INSTR_REGADD_RHM] = instr_reg_export,
10992         [INSTR_REGADD_RMH] = instr_reg_export,
10993         [INSTR_REGADD_RMM] = instr_reg_export,
10994         [INSTR_REGADD_RHI] = instr_reg_export,
10995         [INSTR_REGADD_RMI] = instr_reg_export,
10996         [INSTR_REGADD_RIH] = instr_reg_export,
10997         [INSTR_REGADD_RIM] = instr_reg_export,
10998         [INSTR_REGADD_RII] = instr_reg_export,
10999
11000         [INSTR_METPREFETCH_H] = instr_meter_export,
11001         [INSTR_METPREFETCH_M] = instr_meter_export,
11002         [INSTR_METPREFETCH_I] = instr_meter_export,
11003
11004         [INSTR_METER_HHM] = instr_meter_export,
11005         [INSTR_METER_HHI] = instr_meter_export,
11006         [INSTR_METER_HMM] = instr_meter_export,
11007         [INSTR_METER_HMI] = instr_meter_export,
11008         [INSTR_METER_MHM] = instr_meter_export,
11009         [INSTR_METER_MHI] = instr_meter_export,
11010         [INSTR_METER_MMM] = instr_meter_export,
11011         [INSTR_METER_MMI] = instr_meter_export,
11012         [INSTR_METER_IHM] = instr_meter_export,
11013         [INSTR_METER_IHI] = instr_meter_export,
11014         [INSTR_METER_IMM] = instr_meter_export,
11015         [INSTR_METER_IMI] = instr_meter_export,
11016
11017         [INSTR_TABLE] = instr_table_export,
11018         [INSTR_TABLE_AF] = instr_table_export,
11019         [INSTR_SELECTOR] = instr_table_export,
11020         [INSTR_LEARNER] = instr_table_export,
11021         [INSTR_LEARNER_AF] = instr_table_export,
11022
11023         [INSTR_LEARNER_LEARN] = instr_learn_export,
11024         [INSTR_LEARNER_FORGET] = instr_forget_export,
11025
11026         [INSTR_EXTERN_OBJ] = instr_extern_export,
11027         [INSTR_EXTERN_FUNC] = instr_extern_export,
11028
11029         [INSTR_JMP] = instr_jmp_export,
11030         [INSTR_JMP_VALID] = instr_jmp_export,
11031         [INSTR_JMP_INVALID] = instr_jmp_export,
11032         [INSTR_JMP_HIT] = instr_jmp_export,
11033         [INSTR_JMP_MISS] = instr_jmp_export,
11034         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11035         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11036
11037         [INSTR_JMP_EQ] = instr_jmp_export,
11038         [INSTR_JMP_EQ_MH] = instr_jmp_export,
11039         [INSTR_JMP_EQ_HM] = instr_jmp_export,
11040         [INSTR_JMP_EQ_HH] = instr_jmp_export,
11041         [INSTR_JMP_EQ_I] = instr_jmp_export,
11042
11043         [INSTR_JMP_NEQ] = instr_jmp_export,
11044         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11045         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11046         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11047         [INSTR_JMP_NEQ_I] = instr_jmp_export,
11048
11049         [INSTR_JMP_LT] = instr_jmp_export,
11050         [INSTR_JMP_LT_MH] = instr_jmp_export,
11051         [INSTR_JMP_LT_HM] = instr_jmp_export,
11052         [INSTR_JMP_LT_HH] = instr_jmp_export,
11053         [INSTR_JMP_LT_MI] = instr_jmp_export,
11054         [INSTR_JMP_LT_HI] = instr_jmp_export,
11055
11056         [INSTR_JMP_GT] = instr_jmp_export,
11057         [INSTR_JMP_GT_MH] = instr_jmp_export,
11058         [INSTR_JMP_GT_HM] = instr_jmp_export,
11059         [INSTR_JMP_GT_HH] = instr_jmp_export,
11060         [INSTR_JMP_GT_MI] = instr_jmp_export,
11061         [INSTR_JMP_GT_HI] = instr_jmp_export,
11062
11063         [INSTR_RETURN] = instr_return_export,
11064 };
11065
11066 static void
11067 action_data_codegen(struct action *a, FILE *f)
11068 {
11069         uint32_t i;
11070
11071         fprintf(f,
11072                 "static const struct instruction action_%s_instructions[] = {\n",
11073                 a->name);
11074
11075         for (i = 0; i < a->n_instructions; i++) {
11076                 struct instruction *instr = &a->instructions[i];
11077                 instruction_export_t func = export_table[instr->type];
11078
11079                 func(instr, f);
11080         }
11081
11082         fprintf(f, "};\n");
11083 }
11084
11085 static const char *
11086 instr_type_to_func(struct instruction *instr)
11087 {
11088         switch (instr->type) {
11089         case INSTR_RX: return NULL;
11090
11091         case INSTR_TX: return "__instr_tx_exec";
11092         case INSTR_TX_I: return "__instr_tx_i_exec";
11093         case INSTR_DROP: return "__instr_drop_exec";
11094
11095         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11096         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11097         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11098         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11099         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11100         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11101         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11102         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11103
11104         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11105
11106         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11107
11108         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11109         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11110         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11111         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11112         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11113         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11114         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11115         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11116         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11117
11118         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11119         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11120
11121         case INSTR_MOV: return "__instr_mov_exec";
11122         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11123         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11124         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11125         case INSTR_MOV_I: return "__instr_mov_i_exec";
11126
11127         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11128         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11129         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11130         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11131         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11132         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11133         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11134         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11135
11136         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11137         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11138         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11139         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11140         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11141         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11142
11143         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11144         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11145         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11146         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11147         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11148         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11149
11150         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11151         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11152         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11153         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11154
11155         case INSTR_ALU_AND: return "__instr_alu_and_exec";
11156         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11157         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11158         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11159         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11160
11161         case INSTR_ALU_OR: return "__instr_alu_or_exec";
11162         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11163         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11164         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11165         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11166
11167         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11168         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11169         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11170         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11171         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11172
11173         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11174         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11175         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11176         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11177         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11178         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11179
11180         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11181         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11182         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11183         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11184         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11185         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11186
11187         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11188         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11189         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11190
11191         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11192         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11193         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11194         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11195         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11196         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11197
11198         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11199         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11200         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11201         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11202         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11203         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11204         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11205         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11206         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11207
11208         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11209         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11210         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11211         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11212         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11213         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11214         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11215         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11216         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11217
11218         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11219         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11220         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11221
11222         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11223         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11224         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11225         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11226         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11227         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11228         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11229         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11230         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11231         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11232         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11233         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11234
11235         case INSTR_TABLE: return NULL;
11236         case INSTR_TABLE_AF: return NULL;
11237         case INSTR_SELECTOR: return NULL;
11238         case INSTR_LEARNER: return NULL;
11239         case INSTR_LEARNER_AF: return NULL;
11240
11241         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11242         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11243
11244         case INSTR_EXTERN_OBJ: return NULL;
11245         case INSTR_EXTERN_FUNC: return NULL;
11246
11247         case INSTR_JMP: return NULL;
11248         case INSTR_JMP_VALID: return NULL;
11249         case INSTR_JMP_INVALID: return NULL;
11250         case INSTR_JMP_HIT: return NULL;
11251         case INSTR_JMP_MISS: return NULL;
11252         case INSTR_JMP_ACTION_HIT: return NULL;
11253         case INSTR_JMP_ACTION_MISS: return NULL;
11254         case INSTR_JMP_EQ: return NULL;
11255         case INSTR_JMP_EQ_MH: return NULL;
11256         case INSTR_JMP_EQ_HM: return NULL;
11257         case INSTR_JMP_EQ_HH: return NULL;
11258         case INSTR_JMP_EQ_I: return NULL;
11259         case INSTR_JMP_NEQ: return NULL;
11260         case INSTR_JMP_NEQ_MH: return NULL;
11261         case INSTR_JMP_NEQ_HM: return NULL;
11262         case INSTR_JMP_NEQ_HH: return NULL;
11263         case INSTR_JMP_NEQ_I: return NULL;
11264         case INSTR_JMP_LT: return NULL;
11265         case INSTR_JMP_LT_MH: return NULL;
11266         case INSTR_JMP_LT_HM: return NULL;
11267         case INSTR_JMP_LT_HH: return NULL;
11268         case INSTR_JMP_LT_MI: return NULL;
11269         case INSTR_JMP_LT_HI: return NULL;
11270         case INSTR_JMP_GT: return NULL;
11271         case INSTR_JMP_GT_MH: return NULL;
11272         case INSTR_JMP_GT_HM: return NULL;
11273         case INSTR_JMP_GT_HH: return NULL;
11274         case INSTR_JMP_GT_MI: return NULL;
11275         case INSTR_JMP_GT_HI: return NULL;
11276
11277         case INSTR_RETURN: return NULL;
11278
11279         default: return NULL;
11280         }
11281 }
11282
11283 static void
11284 action_instr_does_tx_codegen(struct action *a,
11285                         uint32_t instr_pos,
11286                         struct instruction *instr,
11287                         FILE *f)
11288 {
11289         fprintf(f,
11290                 "%s(p, t, &action_%s_instructions[%u]);\n"
11291                 "\tthread_ip_reset(p, t);\n"
11292                 "\tinstr_rx_exec(p);\n"
11293                 "\treturn;\n",
11294                 instr_type_to_func(instr),
11295                 a->name,
11296                 instr_pos);
11297 }
11298
11299 static void
11300 action_instr_extern_obj_codegen(struct action *a,
11301                                 uint32_t instr_pos,
11302                                 FILE *f)
11303 {
11304         fprintf(f,
11305                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11306                 a->name,
11307                 instr_pos);
11308 }
11309
11310 static void
11311 action_instr_extern_func_codegen(struct action *a,
11312                                  uint32_t instr_pos,
11313                                  FILE *f)
11314 {
11315         fprintf(f,
11316                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11317                 a->name,
11318                 instr_pos);
11319 }
11320
11321 static void
11322 action_instr_jmp_codegen(struct action *a,
11323                          uint32_t instr_pos,
11324                          struct instruction *instr,
11325                          struct instruction_data *data,
11326                          FILE *f)
11327 {
11328         switch (instr->type) {
11329         case INSTR_JMP:
11330                 fprintf(f,
11331                         "goto %s;\n",
11332                         data->jmp_label);
11333                 return;
11334
11335         case INSTR_JMP_VALID:
11336                 fprintf(f,
11337                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11338                         "\t\tgoto %s;\n",
11339                         a->name,
11340                         instr_pos,
11341                         data->jmp_label);
11342                 return;
11343
11344         case INSTR_JMP_INVALID:
11345                 fprintf(f,
11346                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11347                         "\t\tgoto %s;\n",
11348                         a->name,
11349                         instr_pos,
11350                         data->jmp_label);
11351                 return;
11352
11353         case INSTR_JMP_HIT:
11354                 fprintf(f,
11355                         "if (t->hit)\n"
11356                         "\t\tgoto %s;\n",
11357                         data->jmp_label);
11358                 return;
11359
11360         case INSTR_JMP_MISS:
11361                 fprintf(f,
11362                         "if (!t->hit)\n"
11363                         "\t\tgoto %s;\n",
11364                         data->jmp_label);
11365                 return;
11366
11367         case INSTR_JMP_ACTION_HIT:
11368                 fprintf(f,
11369                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11370                         "\t\tgoto %s;\n",
11371                         a->name,
11372                         instr_pos,
11373                         data->jmp_label);
11374                 return;
11375
11376         case INSTR_JMP_ACTION_MISS:
11377                 fprintf(f,
11378                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11379                         "\t\tgoto %s;\n",
11380                         a->name,
11381                         instr_pos,
11382                         data->jmp_label);
11383                 return;
11384
11385         case INSTR_JMP_EQ:
11386                 fprintf(f,
11387                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11388                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11389                         "\t\tgoto %s;\n",
11390                         a->name,
11391                         instr_pos,
11392                         a->name,
11393                         instr_pos,
11394                         data->jmp_label);
11395                 return;
11396
11397         case INSTR_JMP_EQ_MH:
11398                 fprintf(f,
11399                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11400                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11401                         "\t\tgoto %s;\n",
11402                         a->name,
11403                         instr_pos,
11404                         a->name,
11405                         instr_pos,
11406                         data->jmp_label);
11407                 return;
11408
11409         case INSTR_JMP_EQ_HM:
11410                 fprintf(f,
11411                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11412                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11413                         "\t\tgoto %s;\n",
11414                         a->name,
11415                         instr_pos,
11416                         a->name,
11417                         instr_pos,
11418                         data->jmp_label);
11419                 return;
11420
11421         case INSTR_JMP_EQ_HH:
11422                 fprintf(f,
11423                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11424                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11425                         "\t\tgoto %s;\n",
11426                         a->name,
11427                         instr_pos,
11428                         a->name,
11429                         instr_pos,
11430                         data->jmp_label);
11431                 return;
11432
11433         case INSTR_JMP_EQ_I:
11434                 fprintf(f,
11435                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11436                         "action_%s_instructions[%u].jmp.b_val)\n"
11437                         "\t\tgoto %s;\n",
11438                         a->name,
11439                         instr_pos,
11440                         a->name,
11441                         instr_pos,
11442                         data->jmp_label);
11443                 return;
11444
11445         case INSTR_JMP_NEQ:
11446                 fprintf(f,
11447                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11448                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11449                         "\t\tgoto %s;\n",
11450                         a->name,
11451                         instr_pos,
11452                         a->name,
11453                         instr_pos,
11454                         data->jmp_label);
11455                 return;
11456
11457         case INSTR_JMP_NEQ_MH:
11458                 fprintf(f,
11459                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11460                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11461                         "\t\tgoto %s;\n",
11462                         a->name,
11463                         instr_pos,
11464                         a->name,
11465                         instr_pos,
11466                         data->jmp_label);
11467                 return;
11468
11469         case INSTR_JMP_NEQ_HM:
11470                 fprintf(f,
11471                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11472                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11473                         "\t\tgoto %s;\n",
11474                         a->name,
11475                         instr_pos,
11476                         a->name,
11477                         instr_pos,
11478                         data->jmp_label);
11479                 return;
11480
11481         case INSTR_JMP_NEQ_HH:
11482                 fprintf(f,
11483                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11484                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11485                         "\t\tgoto %s;\n",
11486                         a->name,
11487                         instr_pos,
11488                         a->name,
11489                         instr_pos,
11490                         data->jmp_label);
11491                 return;
11492
11493         case INSTR_JMP_NEQ_I:
11494                 fprintf(f,
11495                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11496                         "action_%s_instructions[%u].jmp.b_val)\n"
11497                         "\t\tgoto %s;\n",
11498                         a->name,
11499                         instr_pos,
11500                         a->name,
11501                         instr_pos,
11502                         data->jmp_label);
11503                 return;
11504
11505         case INSTR_JMP_LT:
11506                 fprintf(f,
11507                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11508                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11509                         "\t\tgoto %s;\n",
11510                         a->name,
11511                         instr_pos,
11512                         a->name,
11513                         instr_pos,
11514                         data->jmp_label);
11515                 return;
11516
11517         case INSTR_JMP_LT_MH:
11518                 fprintf(f,
11519                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11520                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11521                         "\t\tgoto %s;\n",
11522                         a->name,
11523                         instr_pos,
11524                         a->name,
11525                         instr_pos,
11526                         data->jmp_label);
11527                 return;
11528
11529         case INSTR_JMP_LT_HM:
11530                 fprintf(f,
11531                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11532                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11533                         "\t\tgoto %s;\n",
11534                         a->name,
11535                         instr_pos,
11536                         a->name,
11537                         instr_pos,
11538                         data->jmp_label);
11539                 return;
11540
11541         case INSTR_JMP_LT_HH:
11542                 fprintf(f,
11543                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11544                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11545                         "\t\tgoto %s;\n",
11546                         a->name,
11547                         instr_pos,
11548                         a->name,
11549                         instr_pos,
11550                         data->jmp_label);
11551                 return;
11552
11553         case INSTR_JMP_LT_MI:
11554                 fprintf(f,
11555                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11556                         "action_%s_instructions[%u].jmp.b_val)\n"
11557                         "\t\tgoto %s;\n",
11558                         a->name,
11559                         instr_pos,
11560                         a->name,
11561                         instr_pos,
11562                         data->jmp_label);
11563                 return;
11564
11565         case INSTR_JMP_LT_HI:
11566                 fprintf(f,
11567                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11568                         "action_%s_instructions[%u].jmp.b_val)\n"
11569                         "\t\tgoto %s;\n",
11570                         a->name,
11571                         instr_pos,
11572                         a->name,
11573                         instr_pos,
11574                         data->jmp_label);
11575                 return;
11576
11577         case INSTR_JMP_GT:
11578                 fprintf(f,
11579                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11580                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11581                         "\t\tgoto %s;\n",
11582                         a->name,
11583                         instr_pos,
11584                         a->name,
11585                         instr_pos,
11586                         data->jmp_label);
11587                 return;
11588
11589         case INSTR_JMP_GT_MH:
11590                 fprintf(f,
11591                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11592                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11593                         "\t\tgoto %s;\n",
11594                         a->name,
11595                         instr_pos,
11596                         a->name,
11597                         instr_pos,
11598                         data->jmp_label);
11599                 return;
11600
11601         case INSTR_JMP_GT_HM:
11602                 fprintf(f,
11603                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11604                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11605                         "\t\tgoto %s;\n",
11606                         a->name,
11607                         instr_pos,
11608                         a->name,
11609                         instr_pos,
11610                         data->jmp_label);
11611                 return;
11612
11613         case INSTR_JMP_GT_HH:
11614                 fprintf(f,
11615                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11616                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11617                         "\t\tgoto %s;\n",
11618                         a->name,
11619                         instr_pos,
11620                         a->name,
11621                         instr_pos,
11622                         data->jmp_label);
11623                 return;
11624
11625         case INSTR_JMP_GT_MI:
11626                 fprintf(f,
11627                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11628                         "action_%s_instructions[%u].jmp.b_val)\n"
11629                         "\t\tgoto %s;\n",
11630                         a->name,
11631                         instr_pos,
11632                         a->name,
11633                         instr_pos,
11634                         data->jmp_label);
11635                 return;
11636
11637         case INSTR_JMP_GT_HI:
11638                 fprintf(f,
11639                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11640                         "action_%s_instructions[%u].jmp.b_val)\n"
11641                         "\t\tgoto %s;\n",
11642                         a->name,
11643                         instr_pos,
11644                         a->name,
11645                         instr_pos,
11646                         data->jmp_label);
11647                 return;
11648
11649         default:
11650                 return;
11651         }
11652 }
11653
11654 static void
11655 action_instr_return_codegen(FILE *f)
11656 {
11657         fprintf(f,
11658                 "return;\n");
11659 }
11660
11661 static void
11662 action_instr_codegen(struct action *a, FILE *f)
11663 {
11664         uint32_t i;
11665
11666         fprintf(f,
11667                 "void\n"
11668                 "action_%s_run(struct rte_swx_pipeline *p)\n"
11669                 "{\n"
11670                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11671                 "\n",
11672                 a->name);
11673
11674         for (i = 0; i < a->n_instructions; i++) {
11675                 struct instruction *instr = &a->instructions[i];
11676                 struct instruction_data *data = &a->instruction_data[i];
11677
11678                 /* Label, if present. */
11679                 if (data->label[0])
11680                         fprintf(f, "\n%s : ", data->label);
11681                 else
11682                         fprintf(f, "\n\t");
11683
11684                 /* TX instruction type. */
11685                 if (instruction_does_tx(instr)) {
11686                         action_instr_does_tx_codegen(a, i, instr, f);
11687                         continue;
11688                 }
11689
11690                 /* Extern object/function instruction type. */
11691                 if (instr->type == INSTR_EXTERN_OBJ) {
11692                         action_instr_extern_obj_codegen(a, i, f);
11693                         continue;
11694                 }
11695
11696                 if (instr->type == INSTR_EXTERN_FUNC) {
11697                         action_instr_extern_func_codegen(a, i, f);
11698                         continue;
11699                 }
11700
11701                 /* Jump instruction type. */
11702                 if (instruction_is_jmp(instr)) {
11703                         action_instr_jmp_codegen(a, i, instr, data, f);
11704                         continue;
11705                 }
11706
11707                 /* Return instruction type. */
11708                 if (instr->type == INSTR_RETURN) {
11709                         action_instr_return_codegen(f);
11710                         continue;
11711                 }
11712
11713                 /* Any other instruction type. */
11714                 fprintf(f,
11715                         "%s(p, t, &action_%s_instructions[%u]);\n",
11716                         instr_type_to_func(instr),
11717                         a->name,
11718                         i);
11719         }
11720
11721         fprintf(f, "}\n\n");
11722 }
11723
11724 struct instruction_group {
11725         TAILQ_ENTRY(instruction_group) node;
11726
11727         uint32_t group_id;
11728
11729         uint32_t first_instr_id;
11730
11731         uint32_t last_instr_id;
11732
11733         instr_exec_t func;
11734 };
11735
11736 TAILQ_HEAD(instruction_group_list, instruction_group);
11737
11738 static struct instruction_group *
11739 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11740 {
11741         struct instruction_group *g;
11742
11743         TAILQ_FOREACH(g, igl, node)
11744                 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11745                         return g;
11746
11747         return NULL;
11748 }
11749
11750 static void
11751 instruction_group_list_free(struct instruction_group_list *igl)
11752 {
11753         if (!igl)
11754                 return;
11755
11756         for ( ; ; ) {
11757                 struct instruction_group *g;
11758
11759                 g = TAILQ_FIRST(igl);
11760                 if (!g)
11761                         break;
11762
11763                 TAILQ_REMOVE(igl, g, node);
11764                 free(g);
11765         }
11766
11767         free(igl);
11768 }
11769
11770 static struct instruction_group_list *
11771 instruction_group_list_create(struct rte_swx_pipeline *p)
11772 {
11773         struct instruction_group_list *igl = NULL;
11774         struct instruction_group *g = NULL;
11775         uint32_t n_groups = 0, i;
11776
11777         if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11778                 goto error;
11779
11780         /* List init. */
11781         igl = calloc(1, sizeof(struct instruction_group_list));
11782         if (!igl)
11783                 goto error;
11784
11785         TAILQ_INIT(igl);
11786
11787         /* Allocate the first group. */
11788         g = calloc(1, sizeof(struct instruction_group));
11789         if (!g)
11790                 goto error;
11791
11792         /* Iteration 1: Separate the instructions into groups based on the thread yield
11793          * instructions. Do not worry about the jump instructions at this point.
11794          */
11795         for (i = 0; i < p->n_instructions; i++) {
11796                 struct instruction *instr = &p->instructions[i];
11797
11798                 /* Check for thread yield instructions. */
11799                 if (!instruction_does_thread_yield(instr))
11800                         continue;
11801
11802                 /* If the current group contains at least one instruction, then finalize it (with
11803                  * the previous instruction), add it to the list and allocate a new group (that
11804                  * starts with the current instruction).
11805                  */
11806                 if (i - g->first_instr_id) {
11807                         /* Finalize the group. */
11808                         g->last_instr_id = i - 1;
11809
11810                         /* Add the group to the list. Advance the number of groups. */
11811                         TAILQ_INSERT_TAIL(igl, g, node);
11812                         n_groups++;
11813
11814                         /* Allocate a new group. */
11815                         g = calloc(1, sizeof(struct instruction_group));
11816                         if (!g)
11817                                 goto error;
11818
11819                         /* Initialize the new group. */
11820                         g->group_id = n_groups;
11821                         g->first_instr_id = i;
11822                 }
11823
11824                 /* Finalize the current group (with the current instruction, therefore this group
11825                  * contains just the current thread yield instruction), add it to the list and
11826                  * allocate a new group (that starts with the next instruction).
11827                  */
11828
11829                 /* Finalize the group. */
11830                 g->last_instr_id = i;
11831
11832                 /* Add the group to the list. Advance the number of groups. */
11833                 TAILQ_INSERT_TAIL(igl, g, node);
11834                 n_groups++;
11835
11836                 /* Allocate a new group. */
11837                 g = calloc(1, sizeof(struct instruction_group));
11838                 if (!g)
11839                         goto error;
11840
11841                 /* Initialize the new group. */
11842                 g->group_id = n_groups;
11843                 g->first_instr_id = i + 1;
11844         }
11845
11846         /* Handle the last group. */
11847         if (i - g->first_instr_id) {
11848                 /* Finalize the group. */
11849                 g->last_instr_id = i - 1;
11850
11851                 /* Add the group to the list. Advance the number of groups. */
11852                 TAILQ_INSERT_TAIL(igl, g, node);
11853                 n_groups++;
11854         } else
11855                 free(g);
11856
11857         g = NULL;
11858
11859         /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11860          * the destination of a jump instruction located in a different group ("far jump"), then the
11861          * current group has to be split, so that the instruction representing the far jump
11862          * destination is at the start of its group.
11863          */
11864         for ( ; ; ) {
11865                 int is_modified = 0;
11866
11867                 for (i = 0; i < p->n_instructions; i++) {
11868                         struct instruction_data *data = &p->instruction_data[i];
11869                         struct instruction_group *g;
11870                         uint32_t j;
11871
11872                         /* Continue when the current instruction is not a jump destination. */
11873                         if (!data->n_users)
11874                                 continue;
11875
11876                         g = instruction_group_list_group_find(igl, i);
11877                         if (!g)
11878                                 goto error;
11879
11880                         /* Find out all the jump instructions with this destination. */
11881                         for (j = 0; j < p->n_instructions; j++) {
11882                                 struct instruction *jmp_instr = &p->instructions[j];
11883                                 struct instruction_data *jmp_data = &p->instruction_data[j];
11884                                 struct instruction_group *jmp_g, *new_g;
11885
11886                                 /* Continue when not a jump instruction. Even when jump instruction,
11887                                  * continue when the jump destination is not this instruction.
11888                                  */
11889                                 if (!instruction_is_jmp(jmp_instr) ||
11890                                     strcmp(jmp_data->jmp_label, data->label))
11891                                         continue;
11892
11893                                 jmp_g = instruction_group_list_group_find(igl, j);
11894                                 if (!jmp_g)
11895                                         goto error;
11896
11897                                 /* Continue when both the jump instruction and the jump destination
11898                                  * instruction are in the same group. Even when in different groups,
11899                                  * still continue if the jump destination instruction is already the
11900                                  * first instruction of its group.
11901                                  */
11902                                 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11903                                         continue;
11904
11905                                 /* Split the group of the current jump destination instruction to
11906                                  * make this instruction the first instruction of a new group.
11907                                  */
11908                                 new_g = calloc(1, sizeof(struct instruction_group));
11909                                 if (!new_g)
11910                                         goto error;
11911
11912                                 new_g->group_id = n_groups;
11913                                 new_g->first_instr_id = i;
11914                                 new_g->last_instr_id = g->last_instr_id;
11915
11916                                 g->last_instr_id = i - 1;
11917
11918                                 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11919                                 n_groups++;
11920                                 is_modified = 1;
11921
11922                                 /* The decision to split this group (to make the current instruction
11923                                  * the first instruction of a new group) is already taken and fully
11924                                  * implemented, so no need to search for more reasons to do it.
11925                                  */
11926                                 break;
11927                         }
11928                 }
11929
11930                 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11931                  * previously considered local (i.e. the jump destination is in the same group as
11932                  * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11933                  * different group than the jump instruction). Wost case scenario: each instruction
11934                  * that is a jump destination ends up as the first instruction of its group.
11935                  */
11936                 if (!is_modified)
11937                         break;
11938         }
11939
11940         /* Re-assign the group IDs to be in incremental order. */
11941         i = 0;
11942         TAILQ_FOREACH(g, igl, node) {
11943                 g->group_id = i;
11944
11945                 i++;
11946         }
11947
11948         return igl;
11949
11950 error:
11951         instruction_group_list_free(igl);
11952
11953         free(g);
11954
11955         return NULL;
11956 }
11957
11958 static void
11959 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11960                                uint32_t instr_pos,
11961                                struct instruction *instr,
11962                                FILE *f)
11963 {
11964         fprintf(f,
11965                 "%s(p, t, &pipeline_instructions[%u]);\n"
11966                 "\tthread_ip_reset(p, t);\n"
11967                 "\tinstr_rx_exec(p);\n"
11968                 "\treturn;\n",
11969                 instr_type_to_func(instr),
11970                 instr_pos);
11971 }
11972
11973 static int
11974 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11975                            struct instruction_group_list *igl,
11976                            uint32_t jmp_instr_id,
11977                            struct instruction *jmp_instr,
11978                            struct instruction_data *jmp_data,
11979                            FILE *f)
11980 {
11981         struct instruction_group *jmp_g, *g;
11982         struct instruction_data *data;
11983         uint32_t instr_id;
11984
11985         switch (jmp_instr->type) {
11986         case INSTR_JMP:
11987                 break;
11988
11989         case INSTR_JMP_VALID:
11990                 fprintf(f,
11991                         "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11992                         jmp_instr_id);
11993                 break;
11994
11995         case INSTR_JMP_INVALID:
11996                 fprintf(f,
11997                         "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11998                         jmp_instr_id);
11999                 break;
12000
12001         case INSTR_JMP_HIT:
12002                 fprintf(f,
12003                         "if (t->hit)\n");
12004                 break;
12005
12006         case INSTR_JMP_MISS:
12007                 fprintf(f,
12008                         "if (!t->hit)\n");
12009                 break;
12010
12011         case INSTR_JMP_ACTION_HIT:
12012                 fprintf(f,
12013                         "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12014                         jmp_instr_id);
12015                 break;
12016
12017         case INSTR_JMP_ACTION_MISS:
12018                 fprintf(f,
12019                         "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12020                         jmp_instr_id);
12021                 break;
12022
12023         case INSTR_JMP_EQ:
12024                 fprintf(f,
12025                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12026                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12027                         jmp_instr_id,
12028                         jmp_instr_id);
12029                 break;
12030
12031         case INSTR_JMP_EQ_MH:
12032                 fprintf(f,
12033                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12034                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12035                         jmp_instr_id,
12036                         jmp_instr_id);
12037                 break;
12038
12039         case INSTR_JMP_EQ_HM:
12040                 fprintf(f,
12041                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12042                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12043                         jmp_instr_id,
12044                         jmp_instr_id);
12045                 break;
12046
12047         case INSTR_JMP_EQ_HH:
12048                 fprintf(f,
12049                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12050                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12051                         jmp_instr_id,
12052                         jmp_instr_id);
12053                 break;
12054
12055         case INSTR_JMP_EQ_I:
12056                 fprintf(f,
12057                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12058                         "pipeline_instructions[%u].jmp.b_val)",
12059                         jmp_instr_id,
12060                         jmp_instr_id);
12061                 break;
12062
12063         case INSTR_JMP_NEQ:
12064                 fprintf(f,
12065                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12066                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12067                         jmp_instr_id,
12068                         jmp_instr_id);
12069                 break;
12070
12071         case INSTR_JMP_NEQ_MH:
12072                 fprintf(f,
12073                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12074                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12075                         jmp_instr_id,
12076                         jmp_instr_id);
12077                 break;
12078
12079         case INSTR_JMP_NEQ_HM:
12080                 fprintf(f,
12081                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12082                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12083                         jmp_instr_id,
12084                         jmp_instr_id);
12085                 break;
12086
12087         case INSTR_JMP_NEQ_HH:
12088                 fprintf(f,
12089                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12090                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12091                         jmp_instr_id,
12092                         jmp_instr_id);
12093                 break;
12094
12095         case INSTR_JMP_NEQ_I:
12096                 fprintf(f,
12097                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12098                         "pipeline_instructions[%u].jmp.b_val)",
12099                         jmp_instr_id,
12100                         jmp_instr_id);
12101                 break;
12102
12103         case INSTR_JMP_LT:
12104                 fprintf(f,
12105                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12106                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12107                         jmp_instr_id,
12108                         jmp_instr_id);
12109                 break;
12110
12111         case INSTR_JMP_LT_MH:
12112                 fprintf(f,
12113                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12114                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12115                         jmp_instr_id,
12116                         jmp_instr_id);
12117                 break;
12118
12119         case INSTR_JMP_LT_HM:
12120                 fprintf(f,
12121                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12122                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12123                         jmp_instr_id,
12124                         jmp_instr_id);
12125                 break;
12126
12127         case INSTR_JMP_LT_HH:
12128                 fprintf(f,
12129                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12130                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12131                         jmp_instr_id,
12132                         jmp_instr_id);
12133                 break;
12134
12135         case INSTR_JMP_LT_MI:
12136                 fprintf(f,
12137                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12138                         "pipeline_instructions[%u].jmp.b_val)",
12139                         jmp_instr_id,
12140                         jmp_instr_id);
12141                 break;
12142
12143         case INSTR_JMP_LT_HI:
12144                 fprintf(f,
12145                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12146                         "pipeline_instructions[%u].jmp.b_val)",
12147                         jmp_instr_id,
12148                         jmp_instr_id);
12149                 break;
12150
12151         case INSTR_JMP_GT:
12152                 fprintf(f,
12153                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12154                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12155                         jmp_instr_id,
12156                         jmp_instr_id);
12157                 break;
12158
12159         case INSTR_JMP_GT_MH:
12160                 fprintf(f,
12161                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12162                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12163                         jmp_instr_id,
12164                         jmp_instr_id);
12165                 break;
12166
12167         case INSTR_JMP_GT_HM:
12168                 fprintf(f,
12169                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12170                         "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12171                         jmp_instr_id,
12172                         jmp_instr_id);
12173                 break;
12174
12175         case INSTR_JMP_GT_HH:
12176                 fprintf(f,
12177                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12178                         "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12179                         jmp_instr_id,
12180                         jmp_instr_id);
12181                 break;
12182
12183         case INSTR_JMP_GT_MI:
12184                 fprintf(f,
12185                         "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12186                         "pipeline_instructions[%u].jmp.b_val)",
12187                         jmp_instr_id,
12188                         jmp_instr_id);
12189                 break;
12190
12191         case INSTR_JMP_GT_HI:
12192                 fprintf(f,
12193                         "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12194                         "pipeline_instructions[%u].jmp.b_val)",
12195                         jmp_instr_id,
12196                         jmp_instr_id);
12197                 break;
12198
12199         default:
12200                 break;
12201         }
12202
12203         /* Find the instruction group of the jump instruction. */
12204         jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12205         if (!jmp_g)
12206                 return -EINVAL;
12207
12208         /* Find the instruction group of the jump destination instruction. */
12209         data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12210         if (!data)
12211                 return -EINVAL;
12212
12213         instr_id = data - p->instruction_data;
12214
12215         g = instruction_group_list_group_find(igl, instr_id);
12216         if (!g)
12217                 return -EINVAL;
12218
12219         /* Code generation for "near" jump (same instruction group) or "far" jump (different
12220          * instruction group).
12221          */
12222         if (g->group_id == jmp_g->group_id)
12223                 fprintf(f,
12224                         "\n\t\tgoto %s;\n",
12225                         jmp_data->jmp_label);
12226         else
12227                 fprintf(f,
12228                         " {\n"
12229                         "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12230                         "\t\treturn;\n"
12231                         "\t}\n\n",
12232                         g->group_id);
12233
12234         return 0;
12235 }
12236
12237 static void
12238 instruction_group_list_codegen(struct instruction_group_list *igl,
12239                                struct rte_swx_pipeline *p,
12240                                FILE *f)
12241 {
12242         struct instruction_group *g;
12243         uint32_t i;
12244         int is_required = 0;
12245
12246         /* Check if code generation is required. */
12247         TAILQ_FOREACH(g, igl, node)
12248                 if (g->first_instr_id < g->last_instr_id)
12249                         is_required = 1;
12250
12251         if (!is_required)
12252                 return;
12253
12254         /* Generate the code for the pipeline instruction array. */
12255         fprintf(f,
12256                 "static const struct instruction pipeline_instructions[] = {\n");
12257
12258         for (i = 0; i < p->n_instructions; i++) {
12259                 struct instruction *instr = &p->instructions[i];
12260                 instruction_export_t func = export_table[instr->type];
12261
12262                 func(instr, f);
12263         }
12264
12265         fprintf(f, "};\n\n");
12266
12267         /* Generate the code for the pipeline functions: one function for each instruction group
12268          * that contains more than one instruction.
12269          */
12270         TAILQ_FOREACH(g, igl, node) {
12271                 struct instruction *last_instr;
12272                 uint32_t j;
12273
12274                 /* Skip if group contains a single instruction. */
12275                 if (g->last_instr_id == g->first_instr_id)
12276                         continue;
12277
12278                 /* Generate new pipeline function. */
12279                 fprintf(f,
12280                         "void\n"
12281                         "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12282                         "{\n"
12283                         "\tstruct thread *t = &p->threads[p->thread_id];\n"
12284                         "\n",
12285                         g->group_id);
12286
12287                 /* Generate the code for each pipeline instruction. */
12288                 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12289                         struct instruction *instr = &p->instructions[j];
12290                         struct instruction_data *data = &p->instruction_data[j];
12291
12292                         /* Label, if present. */
12293                         if (data->label[0])
12294                                 fprintf(f, "\n%s : ", data->label);
12295                         else
12296                                 fprintf(f, "\n\t");
12297
12298                         /* TX instruction type. */
12299                         if (instruction_does_tx(instr)) {
12300                                 pipeline_instr_does_tx_codegen(p, j, instr, f);
12301                                 continue;
12302                         }
12303
12304                         /* Jump instruction type. */
12305                         if (instruction_is_jmp(instr)) {
12306                                 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12307                                 continue;
12308                         }
12309
12310                         /* Any other instruction type. */
12311                         fprintf(f,
12312                                 "%s(p, t, &pipeline_instructions[%u]);\n",
12313                                 instr_type_to_func(instr),
12314                                 j);
12315                 }
12316
12317                 /* Finalize the generated pipeline function. For some instructions such as TX,
12318                  * emit-many-and-TX and unconditional jump, the next instruction has been already
12319                  * decided unconditionally and the instruction pointer of the current thread set
12320                  * accordingly; for all the other instructions, the instruction pointer must be
12321                  * incremented now.
12322                  */
12323                 last_instr = &p->instructions[g->last_instr_id];
12324
12325                 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12326                         fprintf(f,
12327                                 "thread_ip_inc(p);\n");
12328
12329                 fprintf(f,
12330                         "}\n"
12331                         "\n");
12332         }
12333 }
12334
12335 static uint32_t
12336 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12337 {
12338         struct instruction_group *g;
12339         uint32_t n_custom_instr = 0;
12340
12341         /* Groups with a single instruction: no function is generated for this group, the group
12342          * keeps its current instruction. Groups with more than two instructions: one function and
12343          * the associated custom instruction get generated for each such group.
12344          */
12345         TAILQ_FOREACH(g, igl, node) {
12346                 if (g->first_instr_id == g->last_instr_id)
12347                         continue;
12348
12349                 n_custom_instr++;
12350         }
12351
12352         return n_custom_instr;
12353 }
12354
12355 static int
12356 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12357 {
12358         struct action *a;
12359         FILE *f = NULL;
12360
12361         /* Create the .c file. */
12362         f = fopen("/tmp/pipeline.c", "w");
12363         if (!f)
12364                 return -EIO;
12365
12366         /* Include the .h file. */
12367         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12368
12369         /* Add the code for each action. */
12370         TAILQ_FOREACH(a, &p->actions, node) {
12371                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12372
12373                 action_data_codegen(a, f);
12374
12375                 fprintf(f, "\n");
12376
12377                 action_instr_codegen(a, f);
12378
12379                 fprintf(f, "\n");
12380         }
12381
12382         /* Add the pipeline code. */
12383         instruction_group_list_codegen(igl, p, f);
12384
12385         /* Close the .c file. */
12386         fclose(f);
12387
12388         return 0;
12389 }
12390
12391 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12392 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12393 #endif
12394
12395 static int
12396 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12397 {
12398         struct action *a;
12399         struct instruction_group *g;
12400         char *dir_in, *buffer = NULL;
12401         const char *dir_out;
12402         int status = 0;
12403
12404         /* Get the environment variables. */
12405         dir_in = getenv("RTE_INSTALL_DIR");
12406         if (!dir_in) {
12407                 status = -EINVAL;
12408                 goto free;
12409         }
12410
12411         dir_out = "/tmp";
12412
12413         /* Memory allocation for the command buffer. */
12414         buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12415         if (!buffer) {
12416                 status = -ENOMEM;
12417                 goto free;
12418         }
12419
12420         snprintf(buffer,
12421                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12422                  "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12423                  "-I %s/lib/pipeline "
12424                  "-I %s/lib/eal/include "
12425                  "-I %s/lib/eal/x86/include "
12426                  "-I %s/lib/eal/include/generic "
12427                  "-I %s/lib/meter "
12428                  "-I %s/lib/port "
12429                  "-I %s/lib/table "
12430                  "-I %s/lib/pipeline "
12431                  "-I %s/config "
12432                  "-I %s/build "
12433                  "-I %s/lib/eal/linux/include "
12434                  ">%s/pipeline.log 2>&1 "
12435                  "&& "
12436                  "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12437                  ">>%s/pipeline.log 2>&1",
12438                  dir_out,
12439                  dir_out,
12440                  dir_in,
12441                  dir_in,
12442                  dir_in,
12443                  dir_in,
12444                  dir_in,
12445                  dir_in,
12446                  dir_in,
12447                  dir_in,
12448                  dir_in,
12449                  dir_in,
12450                  dir_in,
12451                  dir_out,
12452                  dir_out,
12453                  dir_out,
12454                  dir_out);
12455
12456         /* Build the shared object library. */
12457         status = system(buffer);
12458         if (status)
12459                 goto free;
12460
12461         /* Open library. */
12462         snprintf(buffer,
12463                  RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12464                  "%s/libpipeline.so",
12465                  dir_out);
12466
12467         p->lib = dlopen(buffer, RTLD_LAZY);
12468         if (!p->lib) {
12469                 status = -EIO;
12470                 goto free;
12471         }
12472
12473         /* Get the action function symbols. */
12474         TAILQ_FOREACH(a, &p->actions, node) {
12475                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12476
12477                 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12478                 if (!p->action_funcs[a->id]) {
12479                         status = -EINVAL;
12480                         goto free;
12481                 }
12482         }
12483
12484         /* Get the pipeline function symbols. */
12485         TAILQ_FOREACH(g, igl, node) {
12486                 if (g->first_instr_id == g->last_instr_id)
12487                         continue;
12488
12489                 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12490
12491                 g->func = dlsym(p->lib, buffer);
12492                 if (!g->func) {
12493                         status = -EINVAL;
12494                         goto free;
12495                 }
12496         }
12497
12498 free:
12499         if (status && p->lib) {
12500                 dlclose(p->lib);
12501                 p->lib = NULL;
12502         }
12503
12504         free(buffer);
12505
12506         return status;
12507 }
12508
12509 static int
12510 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12511                       struct instruction_group_list *igl)
12512 {
12513         uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12514
12515         /* Check that enough space is available within the pipeline instruction table to store all
12516          * the custom instructions.
12517          */
12518         if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12519                 return -ENOSPC;
12520
12521         return 0;
12522 }
12523
12524 static void
12525 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12526 {
12527         struct instruction_group *g;
12528         uint32_t i;
12529
12530         /* Pipeline table instructions. */
12531         for (i = 0; i < p->n_instructions; i++) {
12532                 struct instruction *instr = &p->instructions[i];
12533
12534                 if (instr->type == INSTR_TABLE)
12535                         instr->type = INSTR_TABLE_AF;
12536
12537                 if (instr->type == INSTR_LEARNER)
12538                         instr->type = INSTR_LEARNER_AF;
12539         }
12540
12541         /* Pipeline custom instructions. */
12542         i = 0;
12543         TAILQ_FOREACH(g, igl, node) {
12544                 struct instruction *instr = &p->instructions[g->first_instr_id];
12545                 uint32_t j;
12546
12547                 if (g->first_instr_id == g->last_instr_id)
12548                         continue;
12549
12550                 /* Install a new custom instruction. */
12551                 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12552
12553                 /* First instruction of the group: change its type to the new custom instruction. */
12554                 instr->type = INSTR_CUSTOM_0 + i;
12555
12556                 /* All the subsequent instructions of the group: invalidate. */
12557                 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12558                         struct instruction_data *data = &p->instruction_data[j];
12559
12560                         data->invalid = 1;
12561                 }
12562
12563                 i++;
12564         }
12565
12566         /* Remove the invalidated instructions. */
12567         p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12568
12569         /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12570          * instructions that are the only instruction within their group, so they were left
12571          * unmodified).
12572          */
12573         instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12574 }
12575
12576 static int
12577 pipeline_compile(struct rte_swx_pipeline *p)
12578 {
12579         struct instruction_group_list *igl = NULL;
12580         int status = 0;
12581
12582         igl = instruction_group_list_create(p);
12583         if (!igl) {
12584                 status = -ENOMEM;
12585                 goto free;
12586         }
12587
12588         /* Code generation. */
12589         status = pipeline_codegen(p, igl);
12590         if (status)
12591                 goto free;
12592
12593         /* Build and load the shared object library. */
12594         status = pipeline_libload(p, igl);
12595         if (status)
12596                 goto free;
12597
12598         /* Adjust instructions. */
12599         status = pipeline_adjust_check(p, igl);
12600         if (status)
12601                 goto free;
12602
12603         pipeline_adjust(p, igl);
12604
12605 free:
12606         instruction_group_list_free(igl);
12607
12608         return status;
12609 }