lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <arpa/inet.h>
   8
   9 #include "rte_swx_pipeline_internal.h"
  10
  11 #define CHECK(condition, err_code)                                             \
  12 do {                                                                           \
  13         if (!(condition))                                                      \
  14                 return -(err_code);                                            \
  15 } while (0)
  16
  17 #define CHECK_NAME(name, err_code)                                             \
  18         CHECK((name) &&                                                        \
  19               (name)[0] &&                                                     \
  20               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  21               err_code)
  22
  23 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  24         CHECK((instr) &&                                                       \
  25               (instr)[0] &&                                                    \
  26               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  27                RTE_SWX_INSTRUCTION_SIZE),                                      \
  28               err_code)
  29
  30 /*
  31  * Environment.
  32  */
  33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  34
  35 #include <rte_malloc.h>
  36
  37 static void *
  38 env_malloc(size_t size, size_t alignment, int numa_node)
  39 {
  40         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  41 }
  42
  43 static void
  44 env_free(void *start, size_t size __rte_unused)
  45 {
  46         rte_free(start);
  47 }
  48
  49 #else
  50
  51 #include <numa.h>
  52
  53 static void *
  54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  55 {
  56         void *start;
  57
  58         if (numa_available() == -1)
  59                 return NULL;
  60
  61         start = numa_alloc_onnode(size, numa_node);
  62         if (!start)
  63                 return NULL;
  64
  65         memset(start, 0, size);
  66         return start;
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size)
  71 {
  72         if (numa_available() == -1)
  73                 return;
  74
  75         numa_free(start, size);
  76 }
  77
  78 #endif
  79
  80 /*
  81  * Struct.
  82  */
  83 static struct struct_type *
  84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
  85 {
  86         struct struct_type *elem;
  87
  88         TAILQ_FOREACH(elem, &p->struct_types, node)
  89                 if (strcmp(elem->name, name) == 0)
  90                         return elem;
  91
  92         return NULL;
  93 }
  94
  95 static struct field *
  96 struct_type_field_find(struct struct_type *st, const char *name)
  97 {
  98         uint32_t i;
  99
 100         for (i = 0; i < st->n_fields; i++) {
 101                 struct field *f = &st->fields[i];
 102
 103                 if (strcmp(f->name, name) == 0)
 104                         return f;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 int
 111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 112                                       const char *name,
 113                                       struct rte_swx_field_params *fields,
 114                                       uint32_t n_fields,
 115                                       int last_field_has_variable_size)
 116 {
 117         struct struct_type *st;
 118         uint32_t i;
 119
 120         CHECK(p, EINVAL);
 121         CHECK_NAME(name, EINVAL);
 122         CHECK(fields, EINVAL);
 123         CHECK(n_fields, EINVAL);
 124
 125         for (i = 0; i < n_fields; i++) {
 126                 struct rte_swx_field_params *f = &fields[i];
 127                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 128                 uint32_t j;
 129
 130                 CHECK_NAME(f->name, EINVAL);
 131                 CHECK(f->n_bits, EINVAL);
 132                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
 133                 CHECK((f->n_bits & 7) == 0, EINVAL);
 134
 135                 for (j = 0; j < i; j++) {
 136                         struct rte_swx_field_params *f_prev = &fields[j];
 137
 138                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 139                 }
 140         }
 141
 142         CHECK(!struct_type_find(p, name), EEXIST);
 143
 144         /* Node allocation. */
 145         st = calloc(1, sizeof(struct struct_type));
 146         CHECK(st, ENOMEM);
 147
 148         st->fields = calloc(n_fields, sizeof(struct field));
 149         if (!st->fields) {
 150                 free(st);
 151                 CHECK(0, ENOMEM);
 152         }
 153
 154         /* Node initialization. */
 155         strcpy(st->name, name);
 156         for (i = 0; i < n_fields; i++) {
 157                 struct field *dst = &st->fields[i];
 158                 struct rte_swx_field_params *src = &fields[i];
 159                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
 160
 161                 strcpy(dst->name, src->name);
 162                 dst->n_bits = src->n_bits;
 163                 dst->offset = st->n_bits;
 164                 dst->var_size = var_size;
 165
 166                 st->n_bits += src->n_bits;
 167                 st->n_bits_min += var_size ? 0 : src->n_bits;
 168         }
 169         st->n_fields = n_fields;
 170         st->var_size = last_field_has_variable_size;
 171
 172         /* Node add to tailq. */
 173         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 struct_build(struct rte_swx_pipeline *p)
 180 {
 181         uint32_t i;
 182
 183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 184                 struct thread *t = &p->threads[i];
 185
 186                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 187                 CHECK(t->structs, ENOMEM);
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 struct_build_free(struct rte_swx_pipeline *p)
 195 {
 196         uint32_t i;
 197
 198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 199                 struct thread *t = &p->threads[i];
 200
 201                 free(t->structs);
 202                 t->structs = NULL;
 203         }
 204 }
 205
 206 static void
 207 struct_free(struct rte_swx_pipeline *p)
 208 {
 209         struct_build_free(p);
 210
 211         /* Struct types. */
 212         for ( ; ; ) {
 213                 struct struct_type *elem;
 214
 215                 elem = TAILQ_FIRST(&p->struct_types);
 216                 if (!elem)
 217                         break;
 218
 219                 TAILQ_REMOVE(&p->struct_types, elem, node);
 220                 free(elem->fields);
 221                 free(elem);
 222         }
 223 }
 224
 225 /*
 226  * Input port.
 227  */
 228 static struct port_in_type *
 229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 230 {
 231         struct port_in_type *elem;
 232
 233         if (!name)
 234                 return NULL;
 235
 236         TAILQ_FOREACH(elem, &p->port_in_types, node)
 237                 if (strcmp(elem->name, name) == 0)
 238                         return elem;
 239
 240         return NULL;
 241 }
 242
 243 int
 244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 245                                        const char *name,
 246                                        struct rte_swx_port_in_ops *ops)
 247 {
 248         struct port_in_type *elem;
 249
 250         CHECK(p, EINVAL);
 251         CHECK_NAME(name, EINVAL);
 252         CHECK(ops, EINVAL);
 253         CHECK(ops->create, EINVAL);
 254         CHECK(ops->free, EINVAL);
 255         CHECK(ops->pkt_rx, EINVAL);
 256         CHECK(ops->stats_read, EINVAL);
 257
 258         CHECK(!port_in_type_find(p, name), EEXIST);
 259
 260         /* Node allocation. */
 261         elem = calloc(1, sizeof(struct port_in_type));
 262         CHECK(elem, ENOMEM);
 263
 264         /* Node initialization. */
 265         strcpy(elem->name, name);
 266         memcpy(&elem->ops, ops, sizeof(*ops));
 267
 268         /* Node add to tailq. */
 269         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 270
 271         return 0;
 272 }
 273
 274 static struct port_in *
 275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 276 {
 277         struct port_in *port;
 278
 279         TAILQ_FOREACH(port, &p->ports_in, node)
 280                 if (port->id == port_id)
 281                         return port;
 282
 283         return NULL;
 284 }
 285
 286 int
 287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 288                                 uint32_t port_id,
 289                                 const char *port_type_name,
 290                                 void *args)
 291 {
 292         struct port_in_type *type = NULL;
 293         struct port_in *port = NULL;
 294         void *obj = NULL;
 295
 296         CHECK(p, EINVAL);
 297
 298         CHECK(!port_in_find(p, port_id), EINVAL);
 299
 300         CHECK_NAME(port_type_name, EINVAL);
 301         type = port_in_type_find(p, port_type_name);
 302         CHECK(type, EINVAL);
 303
 304         obj = type->ops.create(args);
 305         CHECK(obj, ENODEV);
 306
 307         /* Node allocation. */
 308         port = calloc(1, sizeof(struct port_in));
 309         CHECK(port, ENOMEM);
 310
 311         /* Node initialization. */
 312         port->type = type;
 313         port->obj = obj;
 314         port->id = port_id;
 315
 316         /* Node add to tailq. */
 317         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 318         if (p->n_ports_in < port_id + 1)
 319                 p->n_ports_in = port_id + 1;
 320
 321         return 0;
 322 }
 323
 324 static int
 325 port_in_build(struct rte_swx_pipeline *p)
 326 {
 327         struct port_in *port;
 328         uint32_t i;
 329
 330         CHECK(p->n_ports_in, EINVAL);
 331         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 332
 333         for (i = 0; i < p->n_ports_in; i++)
 334                 CHECK(port_in_find(p, i), EINVAL);
 335
 336         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 337         CHECK(p->in, ENOMEM);
 338
 339         TAILQ_FOREACH(port, &p->ports_in, node) {
 340                 struct port_in_runtime *in = &p->in[port->id];
 341
 342                 in->pkt_rx = port->type->ops.pkt_rx;
 343                 in->obj = port->obj;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static void
 350 port_in_build_free(struct rte_swx_pipeline *p)
 351 {
 352         free(p->in);
 353         p->in = NULL;
 354 }
 355
 356 static void
 357 port_in_free(struct rte_swx_pipeline *p)
 358 {
 359         port_in_build_free(p);
 360
 361         /* Input ports. */
 362         for ( ; ; ) {
 363                 struct port_in *port;
 364
 365                 port = TAILQ_FIRST(&p->ports_in);
 366                 if (!port)
 367                         break;
 368
 369                 TAILQ_REMOVE(&p->ports_in, port, node);
 370                 port->type->ops.free(port->obj);
 371                 free(port);
 372         }
 373
 374         /* Input port types. */
 375         for ( ; ; ) {
 376                 struct port_in_type *elem;
 377
 378                 elem = TAILQ_FIRST(&p->port_in_types);
 379                 if (!elem)
 380                         break;
 381
 382                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 383                 free(elem);
 384         }
 385 }
 386
 387 /*
 388  * Output port.
 389  */
 390 static struct port_out_type *
 391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 392 {
 393         struct port_out_type *elem;
 394
 395         if (!name)
 396                 return NULL;
 397
 398         TAILQ_FOREACH(elem, &p->port_out_types, node)
 399                 if (!strcmp(elem->name, name))
 400                         return elem;
 401
 402         return NULL;
 403 }
 404
 405 int
 406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 407                                         const char *name,
 408                                         struct rte_swx_port_out_ops *ops)
 409 {
 410         struct port_out_type *elem;
 411
 412         CHECK(p, EINVAL);
 413         CHECK_NAME(name, EINVAL);
 414         CHECK(ops, EINVAL);
 415         CHECK(ops->create, EINVAL);
 416         CHECK(ops->free, EINVAL);
 417         CHECK(ops->pkt_tx, EINVAL);
 418         CHECK(ops->stats_read, EINVAL);
 419
 420         CHECK(!port_out_type_find(p, name), EEXIST);
 421
 422         /* Node allocation. */
 423         elem = calloc(1, sizeof(struct port_out_type));
 424         CHECK(elem, ENOMEM);
 425
 426         /* Node initialization. */
 427         strcpy(elem->name, name);
 428         memcpy(&elem->ops, ops, sizeof(*ops));
 429
 430         /* Node add to tailq. */
 431         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 432
 433         return 0;
 434 }
 435
 436 static struct port_out *
 437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 438 {
 439         struct port_out *port;
 440
 441         TAILQ_FOREACH(port, &p->ports_out, node)
 442                 if (port->id == port_id)
 443                         return port;
 444
 445         return NULL;
 446 }
 447
 448 int
 449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 450                                  uint32_t port_id,
 451                                  const char *port_type_name,
 452                                  void *args)
 453 {
 454         struct port_out_type *type = NULL;
 455         struct port_out *port = NULL;
 456         void *obj = NULL;
 457
 458         CHECK(p, EINVAL);
 459
 460         CHECK(!port_out_find(p, port_id), EINVAL);
 461
 462         CHECK_NAME(port_type_name, EINVAL);
 463         type = port_out_type_find(p, port_type_name);
 464         CHECK(type, EINVAL);
 465
 466         obj = type->ops.create(args);
 467         CHECK(obj, ENODEV);
 468
 469         /* Node allocation. */
 470         port = calloc(1, sizeof(struct port_out));
 471         CHECK(port, ENOMEM);
 472
 473         /* Node initialization. */
 474         port->type = type;
 475         port->obj = obj;
 476         port->id = port_id;
 477
 478         /* Node add to tailq. */
 479         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 480         if (p->n_ports_out < port_id + 1)
 481                 p->n_ports_out = port_id + 1;
 482
 483         return 0;
 484 }
 485
 486 static int
 487 port_out_build(struct rte_swx_pipeline *p)
 488 {
 489         struct port_out *port;
 490         uint32_t i;
 491
 492         CHECK(p->n_ports_out, EINVAL);
 493
 494         for (i = 0; i < p->n_ports_out; i++)
 495                 CHECK(port_out_find(p, i), EINVAL);
 496
 497         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 498         CHECK(p->out, ENOMEM);
 499
 500         TAILQ_FOREACH(port, &p->ports_out, node) {
 501                 struct port_out_runtime *out = &p->out[port->id];
 502
 503                 out->pkt_tx = port->type->ops.pkt_tx;
 504                 out->flush = port->type->ops.flush;
 505                 out->obj = port->obj;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static void
 512 port_out_build_free(struct rte_swx_pipeline *p)
 513 {
 514         free(p->out);
 515         p->out = NULL;
 516 }
 517
 518 static void
 519 port_out_free(struct rte_swx_pipeline *p)
 520 {
 521         port_out_build_free(p);
 522
 523         /* Output ports. */
 524         for ( ; ; ) {
 525                 struct port_out *port;
 526
 527                 port = TAILQ_FIRST(&p->ports_out);
 528                 if (!port)
 529                         break;
 530
 531                 TAILQ_REMOVE(&p->ports_out, port, node);
 532                 port->type->ops.free(port->obj);
 533                 free(port);
 534         }
 535
 536         /* Output port types. */
 537         for ( ; ; ) {
 538                 struct port_out_type *elem;
 539
 540                 elem = TAILQ_FIRST(&p->port_out_types);
 541                 if (!elem)
 542                         break;
 543
 544                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 545                 free(elem);
 546         }
 547 }
 548
 549 /*
 550  * Extern object.
 551  */
 552 static struct extern_type *
 553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 554 {
 555         struct extern_type *elem;
 556
 557         TAILQ_FOREACH(elem, &p->extern_types, node)
 558                 if (strcmp(elem->name, name) == 0)
 559                         return elem;
 560
 561         return NULL;
 562 }
 563
 564 static struct extern_type_member_func *
 565 extern_type_member_func_find(struct extern_type *type, const char *name)
 566 {
 567         struct extern_type_member_func *elem;
 568
 569         TAILQ_FOREACH(elem, &type->funcs, node)
 570                 if (strcmp(elem->name, name) == 0)
 571                         return elem;
 572
 573         return NULL;
 574 }
 575
 576 static struct extern_obj *
 577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct extern_obj *elem;
 580
 581         TAILQ_FOREACH(elem, &p->extern_objs, node)
 582                 if (strcmp(elem->name, name) == 0)
 583                         return elem;
 584
 585         return NULL;
 586 }
 587
 588 static struct extern_type_member_func *
 589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
 590                              const char *name,
 591                              struct extern_obj **obj)
 592 {
 593         struct extern_obj *object;
 594         struct extern_type_member_func *func;
 595         char *object_name, *func_name;
 596
 597         if (name[0] != 'e' || name[1] != '.')
 598                 return NULL;
 599
 600         object_name = strdup(&name[2]);
 601         if (!object_name)
 602                 return NULL;
 603
 604         func_name = strchr(object_name, '.');
 605         if (!func_name) {
 606                 free(object_name);
 607                 return NULL;
 608         }
 609
 610         *func_name = 0;
 611         func_name++;
 612
 613         object = extern_obj_find(p, object_name);
 614         if (!object) {
 615                 free(object_name);
 616                 return NULL;
 617         }
 618
 619         func = extern_type_member_func_find(object->type, func_name);
 620         if (!func) {
 621                 free(object_name);
 622                 return NULL;
 623         }
 624
 625         if (obj)
 626                 *obj = object;
 627
 628         free(object_name);
 629         return func;
 630 }
 631
 632 static struct field *
 633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
 634                                const char *name,
 635                                struct extern_obj **object)
 636 {
 637         struct extern_obj *obj;
 638         struct field *f;
 639         char *obj_name, *field_name;
 640
 641         if ((name[0] != 'e') || (name[1] != '.'))
 642                 return NULL;
 643
 644         obj_name = strdup(&name[2]);
 645         if (!obj_name)
 646                 return NULL;
 647
 648         field_name = strchr(obj_name, '.');
 649         if (!field_name) {
 650                 free(obj_name);
 651                 return NULL;
 652         }
 653
 654         *field_name = 0;
 655         field_name++;
 656
 657         obj = extern_obj_find(p, obj_name);
 658         if (!obj) {
 659                 free(obj_name);
 660                 return NULL;
 661         }
 662
 663         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
 664         if (!f) {
 665                 free(obj_name);
 666                 return NULL;
 667         }
 668
 669         if (object)
 670                 *object = obj;
 671
 672         free(obj_name);
 673         return f;
 674 }
 675
 676 int
 677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 678         const char *name,
 679         const char *mailbox_struct_type_name,
 680         rte_swx_extern_type_constructor_t constructor,
 681         rte_swx_extern_type_destructor_t destructor)
 682 {
 683         struct extern_type *elem;
 684         struct struct_type *mailbox_struct_type;
 685
 686         CHECK(p, EINVAL);
 687
 688         CHECK_NAME(name, EINVAL);
 689         CHECK(!extern_type_find(p, name), EEXIST);
 690
 691         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 692         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 693         CHECK(mailbox_struct_type, EINVAL);
 694         CHECK(!mailbox_struct_type->var_size, EINVAL);
 695
 696         CHECK(constructor, EINVAL);
 697         CHECK(destructor, EINVAL);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct extern_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         elem->mailbox_struct_type = mailbox_struct_type;
 706         elem->constructor = constructor;
 707         elem->destructor = destructor;
 708         TAILQ_INIT(&elem->funcs);
 709
 710         /* Node add to tailq. */
 711         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 712
 713         return 0;
 714 }
 715
 716 int
 717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 718         const char *extern_type_name,
 719         const char *name,
 720         rte_swx_extern_type_member_func_t member_func)
 721 {
 722         struct extern_type *type;
 723         struct extern_type_member_func *type_member;
 724
 725         CHECK(p, EINVAL);
 726
 727         CHECK_NAME(extern_type_name, EINVAL);
 728         type = extern_type_find(p, extern_type_name);
 729         CHECK(type, EINVAL);
 730         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 731
 732         CHECK_NAME(name, EINVAL);
 733         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 734
 735         CHECK(member_func, EINVAL);
 736
 737         /* Node allocation. */
 738         type_member = calloc(1, sizeof(struct extern_type_member_func));
 739         CHECK(type_member, ENOMEM);
 740
 741         /* Node initialization. */
 742         strcpy(type_member->name, name);
 743         type_member->func = member_func;
 744         type_member->id = type->n_funcs;
 745
 746         /* Node add to tailq. */
 747         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 748         type->n_funcs++;
 749
 750         return 0;
 751 }
 752
 753 int
 754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 755                                       const char *extern_type_name,
 756                                       const char *name,
 757                                       const char *args)
 758 {
 759         struct extern_type *type;
 760         struct extern_obj *obj;
 761         void *obj_handle;
 762
 763         CHECK(p, EINVAL);
 764
 765         CHECK_NAME(extern_type_name, EINVAL);
 766         type = extern_type_find(p, extern_type_name);
 767         CHECK(type, EINVAL);
 768
 769         CHECK_NAME(name, EINVAL);
 770         CHECK(!extern_obj_find(p, name), EEXIST);
 771
 772         /* Node allocation. */
 773         obj = calloc(1, sizeof(struct extern_obj));
 774         CHECK(obj, ENOMEM);
 775
 776         /* Object construction. */
 777         obj_handle = type->constructor(args);
 778         if (!obj_handle) {
 779                 free(obj);
 780                 CHECK(0, ENODEV);
 781         }
 782
 783         /* Node initialization. */
 784         strcpy(obj->name, name);
 785         obj->type = type;
 786         obj->obj = obj_handle;
 787         obj->struct_id = p->n_structs;
 788         obj->id = p->n_extern_objs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 792         p->n_extern_objs++;
 793         p->n_structs++;
 794
 795         return 0;
 796 }
 797
 798 static int
 799 extern_obj_build(struct rte_swx_pipeline *p)
 800 {
 801         uint32_t i;
 802
 803         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 804                 struct thread *t = &p->threads[i];
 805                 struct extern_obj *obj;
 806
 807                 t->extern_objs = calloc(p->n_extern_objs,
 808                                         sizeof(struct extern_obj_runtime));
 809                 CHECK(t->extern_objs, ENOMEM);
 810
 811                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 812                         struct extern_obj_runtime *r =
 813                                 &t->extern_objs[obj->id];
 814                         struct extern_type_member_func *func;
 815                         uint32_t mailbox_size =
 816                                 obj->type->mailbox_struct_type->n_bits / 8;
 817
 818                         r->obj = obj->obj;
 819
 820                         r->mailbox = calloc(1, mailbox_size);
 821                         CHECK(r->mailbox, ENOMEM);
 822
 823                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 824                                 r->funcs[func->id] = func->func;
 825
 826                         t->structs[obj->struct_id] = r->mailbox;
 827                 }
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 extern_obj_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840                 uint32_t j;
 841
 842                 if (!t->extern_objs)
 843                         continue;
 844
 845                 for (j = 0; j < p->n_extern_objs; j++) {
 846                         struct extern_obj_runtime *r = &t->extern_objs[j];
 847
 848                         free(r->mailbox);
 849                 }
 850
 851                 free(t->extern_objs);
 852                 t->extern_objs = NULL;
 853         }
 854 }
 855
 856 static void
 857 extern_obj_free(struct rte_swx_pipeline *p)
 858 {
 859         extern_obj_build_free(p);
 860
 861         /* Extern objects. */
 862         for ( ; ; ) {
 863                 struct extern_obj *elem;
 864
 865                 elem = TAILQ_FIRST(&p->extern_objs);
 866                 if (!elem)
 867                         break;
 868
 869                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 870                 if (elem->obj)
 871                         elem->type->destructor(elem->obj);
 872                 free(elem);
 873         }
 874
 875         /* Extern types. */
 876         for ( ; ; ) {
 877                 struct extern_type *elem;
 878
 879                 elem = TAILQ_FIRST(&p->extern_types);
 880                 if (!elem)
 881                         break;
 882
 883                 TAILQ_REMOVE(&p->extern_types, elem, node);
 884
 885                 for ( ; ; ) {
 886                         struct extern_type_member_func *func;
 887
 888                         func = TAILQ_FIRST(&elem->funcs);
 889                         if (!func)
 890                                 break;
 891
 892                         TAILQ_REMOVE(&elem->funcs, func, node);
 893                         free(func);
 894                 }
 895
 896                 free(elem);
 897         }
 898 }
 899
 900 /*
 901  * Extern function.
 902  */
 903 static struct extern_func *
 904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 905 {
 906         struct extern_func *elem;
 907
 908         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 909                 if (strcmp(elem->name, name) == 0)
 910                         return elem;
 911
 912         return NULL;
 913 }
 914
 915 static struct extern_func *
 916 extern_func_parse(struct rte_swx_pipeline *p,
 917                   const char *name)
 918 {
 919         if (name[0] != 'f' || name[1] != '.')
 920                 return NULL;
 921
 922         return extern_func_find(p, &name[2]);
 923 }
 924
 925 static struct field *
 926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
 927                                 const char *name,
 928                                 struct extern_func **function)
 929 {
 930         struct extern_func *func;
 931         struct field *f;
 932         char *func_name, *field_name;
 933
 934         if ((name[0] != 'f') || (name[1] != '.'))
 935                 return NULL;
 936
 937         func_name = strdup(&name[2]);
 938         if (!func_name)
 939                 return NULL;
 940
 941         field_name = strchr(func_name, '.');
 942         if (!field_name) {
 943                 free(func_name);
 944                 return NULL;
 945         }
 946
 947         *field_name = 0;
 948         field_name++;
 949
 950         func = extern_func_find(p, func_name);
 951         if (!func) {
 952                 free(func_name);
 953                 return NULL;
 954         }
 955
 956         f = struct_type_field_find(func->mailbox_struct_type, field_name);
 957         if (!f) {
 958                 free(func_name);
 959                 return NULL;
 960         }
 961
 962         if (function)
 963                 *function = func;
 964
 965         free(func_name);
 966         return f;
 967 }
 968
 969 int
 970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 971                                       const char *name,
 972                                       const char *mailbox_struct_type_name,
 973                                       rte_swx_extern_func_t func)
 974 {
 975         struct extern_func *f;
 976         struct struct_type *mailbox_struct_type;
 977
 978         CHECK(p, EINVAL);
 979
 980         CHECK_NAME(name, EINVAL);
 981         CHECK(!extern_func_find(p, name), EEXIST);
 982
 983         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 984         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 985         CHECK(mailbox_struct_type, EINVAL);
 986         CHECK(!mailbox_struct_type->var_size, EINVAL);
 987
 988         CHECK(func, EINVAL);
 989
 990         /* Node allocation. */
 991         f = calloc(1, sizeof(struct extern_func));
 992         CHECK(func, ENOMEM);
 993
 994         /* Node initialization. */
 995         strcpy(f->name, name);
 996         f->mailbox_struct_type = mailbox_struct_type;
 997         f->func = func;
 998         f->struct_id = p->n_structs;
 999         f->id = p->n_extern_funcs;
1000
1001         /* Node add to tailq. */
1002         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003         p->n_extern_funcs++;
1004         p->n_structs++;
1005
1006         return 0;
1007 }
1008
1009 static int
1010 extern_func_build(struct rte_swx_pipeline *p)
1011 {
1012         uint32_t i;
1013
1014         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015                 struct thread *t = &p->threads[i];
1016                 struct extern_func *func;
1017
1018                 /* Memory allocation. */
1019                 t->extern_funcs = calloc(p->n_extern_funcs,
1020                                          sizeof(struct extern_func_runtime));
1021                 CHECK(t->extern_funcs, ENOMEM);
1022
1023                 /* Extern function. */
1024                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025                         struct extern_func_runtime *r =
1026                                 &t->extern_funcs[func->id];
1027                         uint32_t mailbox_size =
1028                                 func->mailbox_struct_type->n_bits / 8;
1029
1030                         r->func = func->func;
1031
1032                         r->mailbox = calloc(1, mailbox_size);
1033                         CHECK(r->mailbox, ENOMEM);
1034
1035                         t->structs[func->struct_id] = r->mailbox;
1036                 }
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1044 {
1045         uint32_t i;
1046
1047         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048                 struct thread *t = &p->threads[i];
1049                 uint32_t j;
1050
1051                 if (!t->extern_funcs)
1052                         continue;
1053
1054                 for (j = 0; j < p->n_extern_funcs; j++) {
1055                         struct extern_func_runtime *r = &t->extern_funcs[j];
1056
1057                         free(r->mailbox);
1058                 }
1059
1060                 free(t->extern_funcs);
1061                 t->extern_funcs = NULL;
1062         }
1063 }
1064
1065 static void
1066 extern_func_free(struct rte_swx_pipeline *p)
1067 {
1068         extern_func_build_free(p);
1069
1070         for ( ; ; ) {
1071                 struct extern_func *elem;
1072
1073                 elem = TAILQ_FIRST(&p->extern_funcs);
1074                 if (!elem)
1075                         break;
1076
1077                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1078                 free(elem);
1079         }
1080 }
1081
1082 /*
1083  * Header.
1084  */
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1087 {
1088         struct header *elem;
1089
1090         TAILQ_FOREACH(elem, &p->headers, node)
1091                 if (strcmp(elem->name, name) == 0)
1092                         return elem;
1093
1094         return NULL;
1095 }
1096
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1099 {
1100         struct header *elem;
1101
1102         TAILQ_FOREACH(elem, &p->headers, node)
1103                 if (elem->struct_id == struct_id)
1104                         return elem;
1105
1106         return NULL;
1107 }
1108
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1111              const char *name)
1112 {
1113         if (name[0] != 'h' || name[1] != '.')
1114                 return NULL;
1115
1116         return header_find(p, &name[2]);
1117 }
1118
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1121                    const char *name,
1122                    struct header **header)
1123 {
1124         struct header *h;
1125         struct field *f;
1126         char *header_name, *field_name;
1127
1128         if ((name[0] != 'h') || (name[1] != '.'))
1129                 return NULL;
1130
1131         header_name = strdup(&name[2]);
1132         if (!header_name)
1133                 return NULL;
1134
1135         field_name = strchr(header_name, '.');
1136         if (!field_name) {
1137                 free(header_name);
1138                 return NULL;
1139         }
1140
1141         *field_name = 0;
1142         field_name++;
1143
1144         h = header_find(p, header_name);
1145         if (!h) {
1146                 free(header_name);
1147                 return NULL;
1148         }
1149
1150         f = struct_type_field_find(h->st, field_name);
1151         if (!f) {
1152                 free(header_name);
1153                 return NULL;
1154         }
1155
1156         if (header)
1157                 *header = h;
1158
1159         free(header_name);
1160         return f;
1161 }
1162
1163 int
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1165                                         const char *name,
1166                                         const char *struct_type_name)
1167 {
1168         struct struct_type *st;
1169         struct header *h;
1170         size_t n_headers_max;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK_NAME(struct_type_name, EINVAL);
1175
1176         CHECK(!header_find(p, name), EEXIST);
1177
1178         st = struct_type_find(p, struct_type_name);
1179         CHECK(st, EINVAL);
1180
1181         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182         CHECK(p->n_headers < n_headers_max, ENOSPC);
1183
1184         /* Node allocation. */
1185         h = calloc(1, sizeof(struct header));
1186         CHECK(h, ENOMEM);
1187
1188         /* Node initialization. */
1189         strcpy(h->name, name);
1190         h->st = st;
1191         h->struct_id = p->n_structs;
1192         h->id = p->n_headers;
1193
1194         /* Node add to tailq. */
1195         TAILQ_INSERT_TAIL(&p->headers, h, node);
1196         p->n_headers++;
1197         p->n_structs++;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 header_build(struct rte_swx_pipeline *p)
1204 {
1205         struct header *h;
1206         uint32_t n_bytes = 0, i;
1207
1208         TAILQ_FOREACH(h, &p->headers, node) {
1209                 n_bytes += h->st->n_bits / 8;
1210         }
1211
1212         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213                 struct thread *t = &p->threads[i];
1214                 uint32_t offset = 0;
1215
1216                 t->headers = calloc(p->n_headers,
1217                                     sizeof(struct header_runtime));
1218                 CHECK(t->headers, ENOMEM);
1219
1220                 t->headers_out = calloc(p->n_headers,
1221                                         sizeof(struct header_out_runtime));
1222                 CHECK(t->headers_out, ENOMEM);
1223
1224                 t->header_storage = calloc(1, n_bytes);
1225                 CHECK(t->header_storage, ENOMEM);
1226
1227                 t->header_out_storage = calloc(1, n_bytes);
1228                 CHECK(t->header_out_storage, ENOMEM);
1229
1230                 TAILQ_FOREACH(h, &p->headers, node) {
1231                         uint8_t *header_storage;
1232                         uint32_t n_bytes =  h->st->n_bits / 8;
1233
1234                         header_storage = &t->header_storage[offset];
1235                         offset += n_bytes;
1236
1237                         t->headers[h->id].ptr0 = header_storage;
1238                         t->headers[h->id].n_bytes = n_bytes;
1239
1240                         t->structs[h->struct_id] = header_storage;
1241                 }
1242         }
1243
1244         return 0;
1245 }
1246
1247 static void
1248 header_build_free(struct rte_swx_pipeline *p)
1249 {
1250         uint32_t i;
1251
1252         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253                 struct thread *t = &p->threads[i];
1254
1255                 free(t->headers_out);
1256                 t->headers_out = NULL;
1257
1258                 free(t->headers);
1259                 t->headers = NULL;
1260
1261                 free(t->header_out_storage);
1262                 t->header_out_storage = NULL;
1263
1264                 free(t->header_storage);
1265                 t->header_storage = NULL;
1266         }
1267 }
1268
1269 static void
1270 header_free(struct rte_swx_pipeline *p)
1271 {
1272         header_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct header *elem;
1276
1277                 elem = TAILQ_FIRST(&p->headers);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->headers, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Meta-data.
1288  */
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         if (!p->metadata_st)
1293                 return NULL;
1294
1295         if (name[0] != 'm' || name[1] != '.')
1296                 return NULL;
1297
1298         return struct_type_field_find(p->metadata_st, &name[2]);
1299 }
1300
1301 int
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303                                           const char *struct_type_name)
1304 {
1305         struct struct_type *st = NULL;
1306
1307         CHECK(p, EINVAL);
1308
1309         CHECK_NAME(struct_type_name, EINVAL);
1310         st  = struct_type_find(p, struct_type_name);
1311         CHECK(st, EINVAL);
1312         CHECK(!st->var_size, EINVAL);
1313         CHECK(!p->metadata_st, EINVAL);
1314
1315         p->metadata_st = st;
1316         p->metadata_struct_id = p->n_structs;
1317
1318         p->n_structs++;
1319
1320         return 0;
1321 }
1322
1323 static int
1324 metadata_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1327         uint32_t i;
1328
1329         /* Thread-level initialization. */
1330         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331                 struct thread *t = &p->threads[i];
1332                 uint8_t *metadata;
1333
1334                 metadata = calloc(1, n_bytes);
1335                 CHECK(metadata, ENOMEM);
1336
1337                 t->metadata = metadata;
1338                 t->structs[p->metadata_struct_id] = metadata;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static void
1345 metadata_build_free(struct rte_swx_pipeline *p)
1346 {
1347         uint32_t i;
1348
1349         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350                 struct thread *t = &p->threads[i];
1351
1352                 free(t->metadata);
1353                 t->metadata = NULL;
1354         }
1355 }
1356
1357 static void
1358 metadata_free(struct rte_swx_pipeline *p)
1359 {
1360         metadata_build_free(p);
1361 }
1362
1363 /*
1364  * Instruction.
1365  */
1366 static int
1367 instruction_is_tx(enum instruction_type type)
1368 {
1369         switch (type) {
1370         case INSTR_TX:
1371         case INSTR_TX_I:
1372                 return 1;
1373
1374         default:
1375                 return 0;
1376         }
1377 }
1378
1379 static int
1380 instruction_does_tx(struct instruction *instr)
1381 {
1382         switch (instr->type) {
1383         case INSTR_TX:
1384         case INSTR_TX_I:
1385         case INSTR_HDR_EMIT_TX:
1386         case INSTR_HDR_EMIT2_TX:
1387         case INSTR_HDR_EMIT3_TX:
1388         case INSTR_HDR_EMIT4_TX:
1389         case INSTR_HDR_EMIT5_TX:
1390         case INSTR_HDR_EMIT6_TX:
1391         case INSTR_HDR_EMIT7_TX:
1392         case INSTR_HDR_EMIT8_TX:
1393                 return 1;
1394         default:
1395                 return 0;
1396         }
1397 }
1398
1399 static int
1400 instruction_is_jmp(struct instruction *instr)
1401 {
1402         switch (instr->type) {
1403         case INSTR_JMP:
1404         case INSTR_JMP_VALID:
1405         case INSTR_JMP_INVALID:
1406         case INSTR_JMP_HIT:
1407         case INSTR_JMP_MISS:
1408         case INSTR_JMP_ACTION_HIT:
1409         case INSTR_JMP_ACTION_MISS:
1410         case INSTR_JMP_EQ:
1411         case INSTR_JMP_EQ_MH:
1412         case INSTR_JMP_EQ_HM:
1413         case INSTR_JMP_EQ_HH:
1414         case INSTR_JMP_EQ_I:
1415         case INSTR_JMP_NEQ:
1416         case INSTR_JMP_NEQ_MH:
1417         case INSTR_JMP_NEQ_HM:
1418         case INSTR_JMP_NEQ_HH:
1419         case INSTR_JMP_NEQ_I:
1420         case INSTR_JMP_LT:
1421         case INSTR_JMP_LT_MH:
1422         case INSTR_JMP_LT_HM:
1423         case INSTR_JMP_LT_HH:
1424         case INSTR_JMP_LT_MI:
1425         case INSTR_JMP_LT_HI:
1426         case INSTR_JMP_GT:
1427         case INSTR_JMP_GT_MH:
1428         case INSTR_JMP_GT_HM:
1429         case INSTR_JMP_GT_HH:
1430         case INSTR_JMP_GT_MI:
1431         case INSTR_JMP_GT_HI:
1432                 return 1;
1433
1434         default:
1435                 return 0;
1436         }
1437 }
1438
1439 static struct field *
1440 action_field_parse(struct action *action, const char *name);
1441
1442 static struct field *
1443 struct_field_parse(struct rte_swx_pipeline *p,
1444                    struct action *action,
1445                    const char *name,
1446                    uint32_t *struct_id)
1447 {
1448         struct field *f;
1449
1450         switch (name[0]) {
1451         case 'h':
1452         {
1453                 struct header *header;
1454
1455                 f = header_field_parse(p, name, &header);
1456                 if (!f)
1457                         return NULL;
1458
1459                 *struct_id = header->struct_id;
1460                 return f;
1461         }
1462
1463         case 'm':
1464         {
1465                 f = metadata_field_parse(p, name);
1466                 if (!f)
1467                         return NULL;
1468
1469                 *struct_id = p->metadata_struct_id;
1470                 return f;
1471         }
1472
1473         case 't':
1474         {
1475                 if (!action)
1476                         return NULL;
1477
1478                 f = action_field_parse(action, name);
1479                 if (!f)
1480                         return NULL;
1481
1482                 *struct_id = 0;
1483                 return f;
1484         }
1485
1486         case 'e':
1487         {
1488                 struct extern_obj *obj;
1489
1490                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1491                 if (!f)
1492                         return NULL;
1493
1494                 *struct_id = obj->struct_id;
1495                 return f;
1496         }
1497
1498         case 'f':
1499         {
1500                 struct extern_func *func;
1501
1502                 f = extern_func_mailbox_field_parse(p, name, &func);
1503                 if (!f)
1504                         return NULL;
1505
1506                 *struct_id = func->struct_id;
1507                 return f;
1508         }
1509
1510         default:
1511                 return NULL;
1512         }
1513 }
1514
1515 /*
1516  * rx.
1517  */
1518 static int
1519 instr_rx_translate(struct rte_swx_pipeline *p,
1520                    struct action *action,
1521                    char **tokens,
1522                    int n_tokens,
1523                    struct instruction *instr,
1524                    struct instruction_data *data __rte_unused)
1525 {
1526         struct field *f;
1527
1528         CHECK(!action, EINVAL);
1529         CHECK(n_tokens == 2, EINVAL);
1530
1531         f = metadata_field_parse(p, tokens[1]);
1532         CHECK(f, EINVAL);
1533
1534         instr->type = INSTR_RX;
1535         instr->io.io.offset = f->offset / 8;
1536         instr->io.io.n_bits = f->n_bits;
1537         return 0;
1538 }
1539
1540 /*
1541  * tx.
1542  */
1543 static int
1544 instr_tx_translate(struct rte_swx_pipeline *p,
1545                    struct action *action __rte_unused,
1546                    char **tokens,
1547                    int n_tokens,
1548                    struct instruction *instr,
1549                    struct instruction_data *data __rte_unused)
1550 {
1551         char *port = tokens[1];
1552         struct field *f;
1553         uint32_t port_val;
1554
1555         CHECK(n_tokens == 2, EINVAL);
1556
1557         f = metadata_field_parse(p, port);
1558         if (f) {
1559                 instr->type = INSTR_TX;
1560                 instr->io.io.offset = f->offset / 8;
1561                 instr->io.io.n_bits = f->n_bits;
1562                 return 0;
1563         }
1564
1565         /* TX_I. */
1566         port_val = strtoul(port, &port, 0);
1567         CHECK(!port[0], EINVAL);
1568
1569         instr->type = INSTR_TX_I;
1570         instr->io.io.val = port_val;
1571         return 0;
1572 }
1573
1574 static int
1575 instr_drop_translate(struct rte_swx_pipeline *p,
1576                      struct action *action __rte_unused,
1577                      char **tokens __rte_unused,
1578                      int n_tokens,
1579                      struct instruction *instr,
1580                      struct instruction_data *data __rte_unused)
1581 {
1582         CHECK(n_tokens == 1, EINVAL);
1583
1584         /* TX_I. */
1585         instr->type = INSTR_TX_I;
1586         instr->io.io.val = p->n_ports_out - 1;
1587         return 0;
1588 }
1589
1590 static inline void
1591 instr_tx_exec(struct rte_swx_pipeline *p)
1592 {
1593         struct thread *t = &p->threads[p->thread_id];
1594         struct instruction *ip = t->ip;
1595
1596         __instr_tx_exec(p, t, ip);
1597
1598         /* Thread. */
1599         thread_ip_reset(p, t);
1600         instr_rx_exec(p);
1601 }
1602
1603 static inline void
1604 instr_tx_i_exec(struct rte_swx_pipeline *p)
1605 {
1606         struct thread *t = &p->threads[p->thread_id];
1607         struct instruction *ip = t->ip;
1608
1609         __instr_tx_i_exec(p, t, ip);
1610
1611         /* Thread. */
1612         thread_ip_reset(p, t);
1613         instr_rx_exec(p);
1614 }
1615
1616 /*
1617  * extract.
1618  */
1619 static int
1620 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1621                             struct action *action,
1622                             char **tokens,
1623                             int n_tokens,
1624                             struct instruction *instr,
1625                             struct instruction_data *data __rte_unused)
1626 {
1627         struct header *h;
1628
1629         CHECK(!action, EINVAL);
1630         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1631
1632         h = header_parse(p, tokens[1]);
1633         CHECK(h, EINVAL);
1634
1635         if (n_tokens == 2) {
1636                 CHECK(!h->st->var_size, EINVAL);
1637
1638                 instr->type = INSTR_HDR_EXTRACT;
1639                 instr->io.hdr.header_id[0] = h->id;
1640                 instr->io.hdr.struct_id[0] = h->struct_id;
1641                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1642         } else {
1643                 struct field *mf;
1644
1645                 CHECK(h->st->var_size, EINVAL);
1646
1647                 mf = metadata_field_parse(p, tokens[2]);
1648                 CHECK(mf, EINVAL);
1649                 CHECK(!mf->var_size, EINVAL);
1650
1651                 instr->type = INSTR_HDR_EXTRACT_M;
1652                 instr->io.io.offset = mf->offset / 8;
1653                 instr->io.io.n_bits = mf->n_bits;
1654                 instr->io.hdr.header_id[0] = h->id;
1655                 instr->io.hdr.struct_id[0] = h->struct_id;
1656                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1657         }
1658
1659         return 0;
1660 }
1661
1662 static int
1663 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1664                               struct action *action,
1665                               char **tokens,
1666                               int n_tokens,
1667                               struct instruction *instr,
1668                               struct instruction_data *data __rte_unused)
1669 {
1670         struct header *h;
1671
1672         CHECK(!action, EINVAL);
1673         CHECK(n_tokens == 2, EINVAL);
1674
1675         h = header_parse(p, tokens[1]);
1676         CHECK(h, EINVAL);
1677         CHECK(!h->st->var_size, EINVAL);
1678
1679         instr->type = INSTR_HDR_LOOKAHEAD;
1680         instr->io.hdr.header_id[0] = h->id;
1681         instr->io.hdr.struct_id[0] = h->struct_id;
1682         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1683
1684         return 0;
1685 }
1686
1687 static inline void
1688 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1689 {
1690         struct thread *t = &p->threads[p->thread_id];
1691         struct instruction *ip = t->ip;
1692
1693         __instr_hdr_extract_exec(p, t, ip);
1694
1695         /* Thread. */
1696         thread_ip_inc(p);
1697 }
1698
1699 static inline void
1700 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1701 {
1702         struct thread *t = &p->threads[p->thread_id];
1703         struct instruction *ip = t->ip;
1704
1705         __instr_hdr_extract2_exec(p, t, ip);
1706
1707         /* Thread. */
1708         thread_ip_inc(p);
1709 }
1710
1711 static inline void
1712 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1713 {
1714         struct thread *t = &p->threads[p->thread_id];
1715         struct instruction *ip = t->ip;
1716
1717         __instr_hdr_extract3_exec(p, t, ip);
1718
1719         /* Thread. */
1720         thread_ip_inc(p);
1721 }
1722
1723 static inline void
1724 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1725 {
1726         struct thread *t = &p->threads[p->thread_id];
1727         struct instruction *ip = t->ip;
1728
1729         __instr_hdr_extract4_exec(p, t, ip);
1730
1731         /* Thread. */
1732         thread_ip_inc(p);
1733 }
1734
1735 static inline void
1736 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1737 {
1738         struct thread *t = &p->threads[p->thread_id];
1739         struct instruction *ip = t->ip;
1740
1741         __instr_hdr_extract5_exec(p, t, ip);
1742
1743         /* Thread. */
1744         thread_ip_inc(p);
1745 }
1746
1747 static inline void
1748 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1749 {
1750         struct thread *t = &p->threads[p->thread_id];
1751         struct instruction *ip = t->ip;
1752
1753         __instr_hdr_extract6_exec(p, t, ip);
1754
1755         /* Thread. */
1756         thread_ip_inc(p);
1757 }
1758
1759 static inline void
1760 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1761 {
1762         struct thread *t = &p->threads[p->thread_id];
1763         struct instruction *ip = t->ip;
1764
1765         __instr_hdr_extract7_exec(p, t, ip);
1766
1767         /* Thread. */
1768         thread_ip_inc(p);
1769 }
1770
1771 static inline void
1772 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1773 {
1774         struct thread *t = &p->threads[p->thread_id];
1775         struct instruction *ip = t->ip;
1776
1777         __instr_hdr_extract8_exec(p, t, ip);
1778
1779         /* Thread. */
1780         thread_ip_inc(p);
1781 }
1782
1783 static inline void
1784 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1785 {
1786         struct thread *t = &p->threads[p->thread_id];
1787         struct instruction *ip = t->ip;
1788
1789         __instr_hdr_extract_m_exec(p, t, ip);
1790
1791         /* Thread. */
1792         thread_ip_inc(p);
1793 }
1794
1795 static inline void
1796 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1797 {
1798         struct thread *t = &p->threads[p->thread_id];
1799         struct instruction *ip = t->ip;
1800
1801         __instr_hdr_lookahead_exec(p, t, ip);
1802
1803         /* Thread. */
1804         thread_ip_inc(p);
1805 }
1806
1807 /*
1808  * emit.
1809  */
1810 static int
1811 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1812                          struct action *action __rte_unused,
1813                          char **tokens,
1814                          int n_tokens,
1815                          struct instruction *instr,
1816                          struct instruction_data *data __rte_unused)
1817 {
1818         struct header *h;
1819
1820         CHECK(n_tokens == 2, EINVAL);
1821
1822         h = header_parse(p, tokens[1]);
1823         CHECK(h, EINVAL);
1824
1825         instr->type = INSTR_HDR_EMIT;
1826         instr->io.hdr.header_id[0] = h->id;
1827         instr->io.hdr.struct_id[0] = h->struct_id;
1828         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1829         return 0;
1830 }
1831
1832 static inline void
1833 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1834 {
1835         struct thread *t = &p->threads[p->thread_id];
1836         struct instruction *ip = t->ip;
1837
1838         __instr_hdr_emit_exec(p, t, ip);
1839
1840         /* Thread. */
1841         thread_ip_inc(p);
1842 }
1843
1844 static inline void
1845 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1846 {
1847         struct thread *t = &p->threads[p->thread_id];
1848         struct instruction *ip = t->ip;
1849
1850         __instr_hdr_emit_tx_exec(p, t, ip);
1851
1852         /* Thread. */
1853         thread_ip_reset(p, t);
1854         instr_rx_exec(p);
1855 }
1856
1857 static inline void
1858 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1859 {
1860         struct thread *t = &p->threads[p->thread_id];
1861         struct instruction *ip = t->ip;
1862
1863         __instr_hdr_emit2_tx_exec(p, t, ip);
1864
1865         /* Thread. */
1866         thread_ip_reset(p, t);
1867         instr_rx_exec(p);
1868 }
1869
1870 static inline void
1871 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1872 {
1873         struct thread *t = &p->threads[p->thread_id];
1874         struct instruction *ip = t->ip;
1875
1876         __instr_hdr_emit3_tx_exec(p, t, ip);
1877
1878         /* Thread. */
1879         thread_ip_reset(p, t);
1880         instr_rx_exec(p);
1881 }
1882
1883 static inline void
1884 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1885 {
1886         struct thread *t = &p->threads[p->thread_id];
1887         struct instruction *ip = t->ip;
1888
1889         __instr_hdr_emit4_tx_exec(p, t, ip);
1890
1891         /* Thread. */
1892         thread_ip_reset(p, t);
1893         instr_rx_exec(p);
1894 }
1895
1896 static inline void
1897 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1898 {
1899         struct thread *t = &p->threads[p->thread_id];
1900         struct instruction *ip = t->ip;
1901
1902         __instr_hdr_emit5_tx_exec(p, t, ip);
1903
1904         /* Thread. */
1905         thread_ip_reset(p, t);
1906         instr_rx_exec(p);
1907 }
1908
1909 static inline void
1910 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1911 {
1912         struct thread *t = &p->threads[p->thread_id];
1913         struct instruction *ip = t->ip;
1914
1915         __instr_hdr_emit6_tx_exec(p, t, ip);
1916
1917         /* Thread. */
1918         thread_ip_reset(p, t);
1919         instr_rx_exec(p);
1920 }
1921
1922 static inline void
1923 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1924 {
1925         struct thread *t = &p->threads[p->thread_id];
1926         struct instruction *ip = t->ip;
1927
1928         __instr_hdr_emit7_tx_exec(p, t, ip);
1929
1930         /* Thread. */
1931         thread_ip_reset(p, t);
1932         instr_rx_exec(p);
1933 }
1934
1935 static inline void
1936 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1937 {
1938         struct thread *t = &p->threads[p->thread_id];
1939         struct instruction *ip = t->ip;
1940
1941         __instr_hdr_emit8_tx_exec(p, t, ip);
1942
1943         /* Thread. */
1944         thread_ip_reset(p, t);
1945         instr_rx_exec(p);
1946 }
1947
1948 /*
1949  * validate.
1950  */
1951 static int
1952 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1953                              struct action *action __rte_unused,
1954                              char **tokens,
1955                              int n_tokens,
1956                              struct instruction *instr,
1957                              struct instruction_data *data __rte_unused)
1958 {
1959         struct header *h;
1960
1961         CHECK(n_tokens == 2, EINVAL);
1962
1963         h = header_parse(p, tokens[1]);
1964         CHECK(h, EINVAL);
1965
1966         instr->type = INSTR_HDR_VALIDATE;
1967         instr->valid.header_id = h->id;
1968         return 0;
1969 }
1970
1971 static inline void
1972 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1973 {
1974         struct thread *t = &p->threads[p->thread_id];
1975         struct instruction *ip = t->ip;
1976
1977         __instr_hdr_validate_exec(p, t, ip);
1978
1979         /* Thread. */
1980         thread_ip_inc(p);
1981 }
1982
1983 /*
1984  * invalidate.
1985  */
1986 static int
1987 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1988                                struct action *action __rte_unused,
1989                                char **tokens,
1990                                int n_tokens,
1991                                struct instruction *instr,
1992                                struct instruction_data *data __rte_unused)
1993 {
1994         struct header *h;
1995
1996         CHECK(n_tokens == 2, EINVAL);
1997
1998         h = header_parse(p, tokens[1]);
1999         CHECK(h, EINVAL);
2000
2001         instr->type = INSTR_HDR_INVALIDATE;
2002         instr->valid.header_id = h->id;
2003         return 0;
2004 }
2005
2006 static inline void
2007 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2008 {
2009         struct thread *t = &p->threads[p->thread_id];
2010         struct instruction *ip = t->ip;
2011
2012         __instr_hdr_invalidate_exec(p, t, ip);
2013
2014         /* Thread. */
2015         thread_ip_inc(p);
2016 }
2017
2018 /*
2019  * table.
2020  */
2021 static struct table *
2022 table_find(struct rte_swx_pipeline *p, const char *name);
2023
2024 static struct selector *
2025 selector_find(struct rte_swx_pipeline *p, const char *name);
2026
2027 static struct learner *
2028 learner_find(struct rte_swx_pipeline *p, const char *name);
2029
2030 static int
2031 instr_table_translate(struct rte_swx_pipeline *p,
2032                       struct action *action,
2033                       char **tokens,
2034                       int n_tokens,
2035                       struct instruction *instr,
2036                       struct instruction_data *data __rte_unused)
2037 {
2038         struct table *t;
2039         struct selector *s;
2040         struct learner *l;
2041
2042         CHECK(!action, EINVAL);
2043         CHECK(n_tokens == 2, EINVAL);
2044
2045         t = table_find(p, tokens[1]);
2046         if (t) {
2047                 instr->type = INSTR_TABLE;
2048                 instr->table.table_id = t->id;
2049                 return 0;
2050         }
2051
2052         s = selector_find(p, tokens[1]);
2053         if (s) {
2054                 instr->type = INSTR_SELECTOR;
2055                 instr->table.table_id = s->id;
2056                 return 0;
2057         }
2058
2059         l = learner_find(p, tokens[1]);
2060         if (l) {
2061                 instr->type = INSTR_LEARNER;
2062                 instr->table.table_id = l->id;
2063                 return 0;
2064         }
2065
2066         CHECK(0, EINVAL);
2067 }
2068
2069 static inline void
2070 instr_table_exec(struct rte_swx_pipeline *p)
2071 {
2072         struct thread *t = &p->threads[p->thread_id];
2073         struct instruction *ip = t->ip;
2074         uint32_t table_id = ip->table.table_id;
2075         struct rte_swx_table_state *ts = &t->table_state[table_id];
2076         struct table_runtime *table = &t->tables[table_id];
2077         struct table_statistics *stats = &p->table_stats[table_id];
2078         uint64_t action_id, n_pkts_hit, n_pkts_action;
2079         uint8_t *action_data;
2080         int done, hit;
2081
2082         /* Table. */
2083         done = table->func(ts->obj,
2084                            table->mailbox,
2085                            table->key,
2086                            &action_id,
2087                            &action_data,
2088                            &hit);
2089         if (!done) {
2090                 /* Thread. */
2091                 TRACE("[Thread %2u] table %u (not finalized)\n",
2092                       p->thread_id,
2093                       table_id);
2094
2095                 thread_yield(p);
2096                 return;
2097         }
2098
2099         action_id = hit ? action_id : ts->default_action_id;
2100         action_data = hit ? action_data : ts->default_action_data;
2101         n_pkts_hit = stats->n_pkts_hit[hit];
2102         n_pkts_action = stats->n_pkts_action[action_id];
2103
2104         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2105               p->thread_id,
2106               table_id,
2107               hit ? "hit" : "miss",
2108               (uint32_t)action_id);
2109
2110         t->action_id = action_id;
2111         t->structs[0] = action_data;
2112         t->hit = hit;
2113         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2114         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2115
2116         /* Thread. */
2117         thread_ip_action_call(p, t, action_id);
2118 }
2119
2120 static inline void
2121 instr_table_af_exec(struct rte_swx_pipeline *p)
2122 {
2123         struct thread *t = &p->threads[p->thread_id];
2124         struct instruction *ip = t->ip;
2125         uint32_t table_id = ip->table.table_id;
2126         struct rte_swx_table_state *ts = &t->table_state[table_id];
2127         struct table_runtime *table = &t->tables[table_id];
2128         struct table_statistics *stats = &p->table_stats[table_id];
2129         uint64_t action_id, n_pkts_hit, n_pkts_action;
2130         uint8_t *action_data;
2131         action_func_t action_func;
2132         int done, hit;
2133
2134         /* Table. */
2135         done = table->func(ts->obj,
2136                            table->mailbox,
2137                            table->key,
2138                            &action_id,
2139                            &action_data,
2140                            &hit);
2141         if (!done) {
2142                 /* Thread. */
2143                 TRACE("[Thread %2u] table %u (not finalized)\n",
2144                       p->thread_id,
2145                       table_id);
2146
2147                 thread_yield(p);
2148                 return;
2149         }
2150
2151         action_id = hit ? action_id : ts->default_action_id;
2152         action_data = hit ? action_data : ts->default_action_data;
2153         action_func = p->action_funcs[action_id];
2154         n_pkts_hit = stats->n_pkts_hit[hit];
2155         n_pkts_action = stats->n_pkts_action[action_id];
2156
2157         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2158               p->thread_id,
2159               table_id,
2160               hit ? "hit" : "miss",
2161               (uint32_t)action_id);
2162
2163         t->action_id = action_id;
2164         t->structs[0] = action_data;
2165         t->hit = hit;
2166         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2167         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2168
2169         /* Thread. */
2170         thread_ip_inc(p);
2171
2172         /* Action. */
2173         action_func(p);
2174 }
2175
2176 static inline void
2177 instr_selector_exec(struct rte_swx_pipeline *p)
2178 {
2179         struct thread *t = &p->threads[p->thread_id];
2180         struct instruction *ip = t->ip;
2181         uint32_t selector_id = ip->table.table_id;
2182         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2183         struct selector_runtime *selector = &t->selectors[selector_id];
2184         struct selector_statistics *stats = &p->selector_stats[selector_id];
2185         uint64_t n_pkts = stats->n_pkts;
2186         int done;
2187
2188         /* Table. */
2189         done = rte_swx_table_selector_select(ts->obj,
2190                            selector->mailbox,
2191                            selector->group_id_buffer,
2192                            selector->selector_buffer,
2193                            selector->member_id_buffer);
2194         if (!done) {
2195                 /* Thread. */
2196                 TRACE("[Thread %2u] selector %u (not finalized)\n",
2197                       p->thread_id,
2198                       selector_id);
2199
2200                 thread_yield(p);
2201                 return;
2202         }
2203
2204
2205         TRACE("[Thread %2u] selector %u\n",
2206               p->thread_id,
2207               selector_id);
2208
2209         stats->n_pkts = n_pkts + 1;
2210
2211         /* Thread. */
2212         thread_ip_inc(p);
2213 }
2214
2215 static inline void
2216 instr_learner_exec(struct rte_swx_pipeline *p)
2217 {
2218         struct thread *t = &p->threads[p->thread_id];
2219         struct instruction *ip = t->ip;
2220         uint32_t learner_id = ip->table.table_id;
2221         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2222                 p->n_selectors + learner_id];
2223         struct learner_runtime *l = &t->learners[learner_id];
2224         struct learner_statistics *stats = &p->learner_stats[learner_id];
2225         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2226         uint8_t *action_data;
2227         int done, hit;
2228
2229         /* Table. */
2230         time = rte_get_tsc_cycles();
2231
2232         done = rte_swx_table_learner_lookup(ts->obj,
2233                                             l->mailbox,
2234                                             time,
2235                                             l->key,
2236                                             &action_id,
2237                                             &action_data,
2238                                             &hit);
2239         if (!done) {
2240                 /* Thread. */
2241                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2242                       p->thread_id,
2243                       learner_id);
2244
2245                 thread_yield(p);
2246                 return;
2247         }
2248
2249         action_id = hit ? action_id : ts->default_action_id;
2250         action_data = hit ? action_data : ts->default_action_data;
2251         n_pkts_hit = stats->n_pkts_hit[hit];
2252         n_pkts_action = stats->n_pkts_action[action_id];
2253
2254         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2255               p->thread_id,
2256               learner_id,
2257               hit ? "hit" : "miss",
2258               (uint32_t)action_id);
2259
2260         t->action_id = action_id;
2261         t->structs[0] = action_data;
2262         t->hit = hit;
2263         t->learner_id = learner_id;
2264         t->time = time;
2265         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2266         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2267
2268         /* Thread. */
2269         thread_ip_action_call(p, t, action_id);
2270 }
2271
2272 static inline void
2273 instr_learner_af_exec(struct rte_swx_pipeline *p)
2274 {
2275         struct thread *t = &p->threads[p->thread_id];
2276         struct instruction *ip = t->ip;
2277         uint32_t learner_id = ip->table.table_id;
2278         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2279                 p->n_selectors + learner_id];
2280         struct learner_runtime *l = &t->learners[learner_id];
2281         struct learner_statistics *stats = &p->learner_stats[learner_id];
2282         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2283         uint8_t *action_data;
2284         action_func_t action_func;
2285         int done, hit;
2286
2287         /* Table. */
2288         time = rte_get_tsc_cycles();
2289
2290         done = rte_swx_table_learner_lookup(ts->obj,
2291                                             l->mailbox,
2292                                             time,
2293                                             l->key,
2294                                             &action_id,
2295                                             &action_data,
2296                                             &hit);
2297         if (!done) {
2298                 /* Thread. */
2299                 TRACE("[Thread %2u] learner %u (not finalized)\n",
2300                       p->thread_id,
2301                       learner_id);
2302
2303                 thread_yield(p);
2304                 return;
2305         }
2306
2307         action_id = hit ? action_id : ts->default_action_id;
2308         action_data = hit ? action_data : ts->default_action_data;
2309         action_func = p->action_funcs[action_id];
2310         n_pkts_hit = stats->n_pkts_hit[hit];
2311         n_pkts_action = stats->n_pkts_action[action_id];
2312
2313         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2314               p->thread_id,
2315               learner_id,
2316               hit ? "hit" : "miss",
2317               (uint32_t)action_id);
2318
2319         t->action_id = action_id;
2320         t->structs[0] = action_data;
2321         t->hit = hit;
2322         t->learner_id = learner_id;
2323         t->time = time;
2324         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2325         stats->n_pkts_action[action_id] = n_pkts_action + 1;
2326
2327         /* Thread. */
2328         thread_ip_action_call(p, t, action_id);
2329
2330         /* Action */
2331         action_func(p);
2332 }
2333
2334 /*
2335  * learn.
2336  */
2337 static struct action *
2338 action_find(struct rte_swx_pipeline *p, const char *name);
2339
2340 static int
2341 action_has_nbo_args(struct action *a);
2342
2343 static int
2344 instr_learn_translate(struct rte_swx_pipeline *p,
2345                       struct action *action,
2346                       char **tokens,
2347                       int n_tokens,
2348                       struct instruction *instr,
2349                       struct instruction_data *data __rte_unused)
2350 {
2351         struct action *a;
2352
2353         CHECK(action, EINVAL);
2354         CHECK(n_tokens == 2, EINVAL);
2355
2356         a = action_find(p, tokens[1]);
2357         CHECK(a, EINVAL);
2358         CHECK(!action_has_nbo_args(a), EINVAL);
2359
2360         instr->type = INSTR_LEARNER_LEARN;
2361         instr->learn.action_id = a->id;
2362
2363         return 0;
2364 }
2365
2366 static inline void
2367 instr_learn_exec(struct rte_swx_pipeline *p)
2368 {
2369         struct thread *t = &p->threads[p->thread_id];
2370         struct instruction *ip = t->ip;
2371
2372         __instr_learn_exec(p, t, ip);
2373
2374         /* Thread. */
2375         thread_ip_inc(p);
2376 }
2377
2378 /*
2379  * forget.
2380  */
2381 static int
2382 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2383                        struct action *action,
2384                        char **tokens __rte_unused,
2385                        int n_tokens,
2386                        struct instruction *instr,
2387                        struct instruction_data *data __rte_unused)
2388 {
2389         CHECK(action, EINVAL);
2390         CHECK(n_tokens == 1, EINVAL);
2391
2392         instr->type = INSTR_LEARNER_FORGET;
2393
2394         return 0;
2395 }
2396
2397 static inline void
2398 instr_forget_exec(struct rte_swx_pipeline *p)
2399 {
2400         struct thread *t = &p->threads[p->thread_id];
2401         struct instruction *ip = t->ip;
2402
2403         __instr_forget_exec(p, t, ip);
2404
2405         /* Thread. */
2406         thread_ip_inc(p);
2407 }
2408
2409 /*
2410  * extern.
2411  */
2412 static int
2413 instr_extern_translate(struct rte_swx_pipeline *p,
2414                        struct action *action __rte_unused,
2415                        char **tokens,
2416                        int n_tokens,
2417                        struct instruction *instr,
2418                        struct instruction_data *data __rte_unused)
2419 {
2420         char *token = tokens[1];
2421
2422         CHECK(n_tokens == 2, EINVAL);
2423
2424         if (token[0] == 'e') {
2425                 struct extern_obj *obj;
2426                 struct extern_type_member_func *func;
2427
2428                 func = extern_obj_member_func_parse(p, token, &obj);
2429                 CHECK(func, EINVAL);
2430
2431                 instr->type = INSTR_EXTERN_OBJ;
2432                 instr->ext_obj.ext_obj_id = obj->id;
2433                 instr->ext_obj.func_id = func->id;
2434
2435                 return 0;
2436         }
2437
2438         if (token[0] == 'f') {
2439                 struct extern_func *func;
2440
2441                 func = extern_func_parse(p, token);
2442                 CHECK(func, EINVAL);
2443
2444                 instr->type = INSTR_EXTERN_FUNC;
2445                 instr->ext_func.ext_func_id = func->id;
2446
2447                 return 0;
2448         }
2449
2450         CHECK(0, EINVAL);
2451 }
2452
2453 static inline void
2454 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2455 {
2456         struct thread *t = &p->threads[p->thread_id];
2457         struct instruction *ip = t->ip;
2458         uint32_t done;
2459
2460         /* Extern object member function execute. */
2461         done = __instr_extern_obj_exec(p, t, ip);
2462
2463         /* Thread. */
2464         thread_ip_inc_cond(t, done);
2465         thread_yield_cond(p, done ^ 1);
2466 }
2467
2468 static inline void
2469 instr_extern_func_exec(struct rte_swx_pipeline *p)
2470 {
2471         struct thread *t = &p->threads[p->thread_id];
2472         struct instruction *ip = t->ip;
2473         uint32_t done;
2474
2475         /* Extern function execute. */
2476         done = __instr_extern_func_exec(p, t, ip);
2477
2478         /* Thread. */
2479         thread_ip_inc_cond(t, done);
2480         thread_yield_cond(p, done ^ 1);
2481 }
2482
2483 /*
2484  * mov.
2485  */
2486 static int
2487 instr_mov_translate(struct rte_swx_pipeline *p,
2488                     struct action *action,
2489                     char **tokens,
2490                     int n_tokens,
2491                     struct instruction *instr,
2492                     struct instruction_data *data __rte_unused)
2493 {
2494         char *dst = tokens[1], *src = tokens[2];
2495         struct field *fdst, *fsrc;
2496         uint64_t src_val;
2497         uint32_t dst_struct_id = 0, src_struct_id = 0;
2498
2499         CHECK(n_tokens == 3, EINVAL);
2500
2501         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2502         CHECK(fdst, EINVAL);
2503         CHECK(!fdst->var_size, EINVAL);
2504
2505         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2506         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2507         if (fsrc) {
2508                 CHECK(!fsrc->var_size, EINVAL);
2509
2510                 instr->type = INSTR_MOV;
2511                 if (dst[0] != 'h' && src[0] == 'h')
2512                         instr->type = INSTR_MOV_MH;
2513                 if (dst[0] == 'h' && src[0] != 'h')
2514                         instr->type = INSTR_MOV_HM;
2515                 if (dst[0] == 'h' && src[0] == 'h')
2516                         instr->type = INSTR_MOV_HH;
2517
2518                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2519                 instr->mov.dst.n_bits = fdst->n_bits;
2520                 instr->mov.dst.offset = fdst->offset / 8;
2521                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2522                 instr->mov.src.n_bits = fsrc->n_bits;
2523                 instr->mov.src.offset = fsrc->offset / 8;
2524                 return 0;
2525         }
2526
2527         /* MOV_I. */
2528         src_val = strtoull(src, &src, 0);
2529         CHECK(!src[0], EINVAL);
2530
2531         if (dst[0] == 'h')
2532                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2533
2534         instr->type = INSTR_MOV_I;
2535         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2536         instr->mov.dst.n_bits = fdst->n_bits;
2537         instr->mov.dst.offset = fdst->offset / 8;
2538         instr->mov.src_val = src_val;
2539         return 0;
2540 }
2541
2542 static inline void
2543 instr_mov_exec(struct rte_swx_pipeline *p)
2544 {
2545         struct thread *t = &p->threads[p->thread_id];
2546         struct instruction *ip = t->ip;
2547
2548         __instr_mov_exec(p, t, ip);
2549
2550         /* Thread. */
2551         thread_ip_inc(p);
2552 }
2553
2554 static inline void
2555 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2556 {
2557         struct thread *t = &p->threads[p->thread_id];
2558         struct instruction *ip = t->ip;
2559
2560         __instr_mov_mh_exec(p, t, ip);
2561
2562         /* Thread. */
2563         thread_ip_inc(p);
2564 }
2565
2566 static inline void
2567 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2568 {
2569         struct thread *t = &p->threads[p->thread_id];
2570         struct instruction *ip = t->ip;
2571
2572         __instr_mov_hm_exec(p, t, ip);
2573
2574         /* Thread. */
2575         thread_ip_inc(p);
2576 }
2577
2578 static inline void
2579 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2580 {
2581         struct thread *t = &p->threads[p->thread_id];
2582         struct instruction *ip = t->ip;
2583
2584         __instr_mov_hh_exec(p, t, ip);
2585
2586         /* Thread. */
2587         thread_ip_inc(p);
2588 }
2589
2590 static inline void
2591 instr_mov_i_exec(struct rte_swx_pipeline *p)
2592 {
2593         struct thread *t = &p->threads[p->thread_id];
2594         struct instruction *ip = t->ip;
2595
2596         __instr_mov_i_exec(p, t, ip);
2597
2598         /* Thread. */
2599         thread_ip_inc(p);
2600 }
2601
2602 /*
2603  * dma.
2604  */
2605 static inline void
2606 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2607 {
2608         struct thread *t = &p->threads[p->thread_id];
2609         struct instruction *ip = t->ip;
2610
2611         __instr_dma_ht_exec(p, t, ip);
2612
2613         /* Thread. */
2614         thread_ip_inc(p);
2615 }
2616
2617 static inline void
2618 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2619 {
2620         struct thread *t = &p->threads[p->thread_id];
2621         struct instruction *ip = t->ip;
2622
2623         __instr_dma_ht2_exec(p, t, ip);
2624
2625         /* Thread. */
2626         thread_ip_inc(p);
2627 }
2628
2629 static inline void
2630 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2631 {
2632         struct thread *t = &p->threads[p->thread_id];
2633         struct instruction *ip = t->ip;
2634
2635         __instr_dma_ht3_exec(p, t, ip);
2636
2637         /* Thread. */
2638         thread_ip_inc(p);
2639 }
2640
2641 static inline void
2642 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2643 {
2644         struct thread *t = &p->threads[p->thread_id];
2645         struct instruction *ip = t->ip;
2646
2647         __instr_dma_ht4_exec(p, t, ip);
2648
2649         /* Thread. */
2650         thread_ip_inc(p);
2651 }
2652
2653 static inline void
2654 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2655 {
2656         struct thread *t = &p->threads[p->thread_id];
2657         struct instruction *ip = t->ip;
2658
2659         __instr_dma_ht5_exec(p, t, ip);
2660
2661         /* Thread. */
2662         thread_ip_inc(p);
2663 }
2664
2665 static inline void
2666 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2667 {
2668         struct thread *t = &p->threads[p->thread_id];
2669         struct instruction *ip = t->ip;
2670
2671         __instr_dma_ht6_exec(p, t, ip);
2672
2673         /* Thread. */
2674         thread_ip_inc(p);
2675 }
2676
2677 static inline void
2678 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2679 {
2680         struct thread *t = &p->threads[p->thread_id];
2681         struct instruction *ip = t->ip;
2682
2683         __instr_dma_ht7_exec(p, t, ip);
2684
2685         /* Thread. */
2686         thread_ip_inc(p);
2687 }
2688
2689 static inline void
2690 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2691 {
2692         struct thread *t = &p->threads[p->thread_id];
2693         struct instruction *ip = t->ip;
2694
2695         __instr_dma_ht8_exec(p, t, ip);
2696
2697         /* Thread. */
2698         thread_ip_inc(p);
2699 }
2700
2701 /*
2702  * alu.
2703  */
2704 static int
2705 instr_alu_add_translate(struct rte_swx_pipeline *p,
2706                         struct action *action,
2707                         char **tokens,
2708                         int n_tokens,
2709                         struct instruction *instr,
2710                         struct instruction_data *data __rte_unused)
2711 {
2712         char *dst = tokens[1], *src = tokens[2];
2713         struct field *fdst, *fsrc;
2714         uint64_t src_val;
2715         uint32_t dst_struct_id = 0, src_struct_id = 0;
2716
2717         CHECK(n_tokens == 3, EINVAL);
2718
2719         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2720         CHECK(fdst, EINVAL);
2721         CHECK(!fdst->var_size, EINVAL);
2722
2723         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2724         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2725         if (fsrc) {
2726                 CHECK(!fsrc->var_size, EINVAL);
2727
2728                 instr->type = INSTR_ALU_ADD;
2729                 if (dst[0] == 'h' && src[0] != 'h')
2730                         instr->type = INSTR_ALU_ADD_HM;
2731                 if (dst[0] != 'h' && src[0] == 'h')
2732                         instr->type = INSTR_ALU_ADD_MH;
2733                 if (dst[0] == 'h' && src[0] == 'h')
2734                         instr->type = INSTR_ALU_ADD_HH;
2735
2736                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2737                 instr->alu.dst.n_bits = fdst->n_bits;
2738                 instr->alu.dst.offset = fdst->offset / 8;
2739                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2740                 instr->alu.src.n_bits = fsrc->n_bits;
2741                 instr->alu.src.offset = fsrc->offset / 8;
2742                 return 0;
2743         }
2744
2745         /* ADD_MI, ADD_HI. */
2746         src_val = strtoull(src, &src, 0);
2747         CHECK(!src[0], EINVAL);
2748
2749         instr->type = INSTR_ALU_ADD_MI;
2750         if (dst[0] == 'h')
2751                 instr->type = INSTR_ALU_ADD_HI;
2752
2753         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2754         instr->alu.dst.n_bits = fdst->n_bits;
2755         instr->alu.dst.offset = fdst->offset / 8;
2756         instr->alu.src_val = src_val;
2757         return 0;
2758 }
2759
2760 static int
2761 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2762                         struct action *action,
2763                         char **tokens,
2764                         int n_tokens,
2765                         struct instruction *instr,
2766                         struct instruction_data *data __rte_unused)
2767 {
2768         char *dst = tokens[1], *src = tokens[2];
2769         struct field *fdst, *fsrc;
2770         uint64_t src_val;
2771         uint32_t dst_struct_id = 0, src_struct_id = 0;
2772
2773         CHECK(n_tokens == 3, EINVAL);
2774
2775         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2776         CHECK(fdst, EINVAL);
2777         CHECK(!fdst->var_size, EINVAL);
2778
2779         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2780         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2781         if (fsrc) {
2782                 CHECK(!fsrc->var_size, EINVAL);
2783
2784                 instr->type = INSTR_ALU_SUB;
2785                 if (dst[0] == 'h' && src[0] != 'h')
2786                         instr->type = INSTR_ALU_SUB_HM;
2787                 if (dst[0] != 'h' && src[0] == 'h')
2788                         instr->type = INSTR_ALU_SUB_MH;
2789                 if (dst[0] == 'h' && src[0] == 'h')
2790                         instr->type = INSTR_ALU_SUB_HH;
2791
2792                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2793                 instr->alu.dst.n_bits = fdst->n_bits;
2794                 instr->alu.dst.offset = fdst->offset / 8;
2795                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2796                 instr->alu.src.n_bits = fsrc->n_bits;
2797                 instr->alu.src.offset = fsrc->offset / 8;
2798                 return 0;
2799         }
2800
2801         /* SUB_MI, SUB_HI. */
2802         src_val = strtoull(src, &src, 0);
2803         CHECK(!src[0], EINVAL);
2804
2805         instr->type = INSTR_ALU_SUB_MI;
2806         if (dst[0] == 'h')
2807                 instr->type = INSTR_ALU_SUB_HI;
2808
2809         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2810         instr->alu.dst.n_bits = fdst->n_bits;
2811         instr->alu.dst.offset = fdst->offset / 8;
2812         instr->alu.src_val = src_val;
2813         return 0;
2814 }
2815
2816 static int
2817 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2818                           struct action *action __rte_unused,
2819                           char **tokens,
2820                           int n_tokens,
2821                           struct instruction *instr,
2822                           struct instruction_data *data __rte_unused)
2823 {
2824         char *dst = tokens[1], *src = tokens[2];
2825         struct header *hdst, *hsrc;
2826         struct field *fdst, *fsrc;
2827
2828         CHECK(n_tokens == 3, EINVAL);
2829
2830         fdst = header_field_parse(p, dst, &hdst);
2831         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2832         CHECK(!fdst->var_size, EINVAL);
2833
2834         /* CKADD_FIELD. */
2835         fsrc = header_field_parse(p, src, &hsrc);
2836         if (fsrc) {
2837                 CHECK(!fsrc->var_size, EINVAL);
2838
2839                 instr->type = INSTR_ALU_CKADD_FIELD;
2840                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2841                 instr->alu.dst.n_bits = fdst->n_bits;
2842                 instr->alu.dst.offset = fdst->offset / 8;
2843                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2844                 instr->alu.src.n_bits = fsrc->n_bits;
2845                 instr->alu.src.offset = fsrc->offset / 8;
2846                 return 0;
2847         }
2848
2849         /* CKADD_STRUCT, CKADD_STRUCT20. */
2850         hsrc = header_parse(p, src);
2851         CHECK(hsrc, EINVAL);
2852         CHECK(!hsrc->st->var_size, EINVAL);
2853
2854         instr->type = INSTR_ALU_CKADD_STRUCT;
2855         if ((hsrc->st->n_bits / 8) == 20)
2856                 instr->type = INSTR_ALU_CKADD_STRUCT20;
2857
2858         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2859         instr->alu.dst.n_bits = fdst->n_bits;
2860         instr->alu.dst.offset = fdst->offset / 8;
2861         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2862         instr->alu.src.n_bits = hsrc->st->n_bits;
2863         instr->alu.src.offset = 0; /* Unused. */
2864         return 0;
2865 }
2866
2867 static int
2868 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2869                           struct action *action __rte_unused,
2870                           char **tokens,
2871                           int n_tokens,
2872                           struct instruction *instr,
2873                           struct instruction_data *data __rte_unused)
2874 {
2875         char *dst = tokens[1], *src = tokens[2];
2876         struct header *hdst, *hsrc;
2877         struct field *fdst, *fsrc;
2878
2879         CHECK(n_tokens == 3, EINVAL);
2880
2881         fdst = header_field_parse(p, dst, &hdst);
2882         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2883         CHECK(!fdst->var_size, EINVAL);
2884
2885         fsrc = header_field_parse(p, src, &hsrc);
2886         CHECK(fsrc, EINVAL);
2887         CHECK(!fsrc->var_size, EINVAL);
2888
2889         instr->type = INSTR_ALU_CKSUB_FIELD;
2890         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2891         instr->alu.dst.n_bits = fdst->n_bits;
2892         instr->alu.dst.offset = fdst->offset / 8;
2893         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2894         instr->alu.src.n_bits = fsrc->n_bits;
2895         instr->alu.src.offset = fsrc->offset / 8;
2896         return 0;
2897 }
2898
2899 static int
2900 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2901                         struct action *action,
2902                         char **tokens,
2903                         int n_tokens,
2904                         struct instruction *instr,
2905                         struct instruction_data *data __rte_unused)
2906 {
2907         char *dst = tokens[1], *src = tokens[2];
2908         struct field *fdst, *fsrc;
2909         uint64_t src_val;
2910         uint32_t dst_struct_id = 0, src_struct_id = 0;
2911
2912         CHECK(n_tokens == 3, EINVAL);
2913
2914         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2915         CHECK(fdst, EINVAL);
2916         CHECK(!fdst->var_size, EINVAL);
2917
2918         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2919         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2920         if (fsrc) {
2921                 CHECK(!fsrc->var_size, EINVAL);
2922
2923                 instr->type = INSTR_ALU_SHL;
2924                 if (dst[0] == 'h' && src[0] != 'h')
2925                         instr->type = INSTR_ALU_SHL_HM;
2926                 if (dst[0] != 'h' && src[0] == 'h')
2927                         instr->type = INSTR_ALU_SHL_MH;
2928                 if (dst[0] == 'h' && src[0] == 'h')
2929                         instr->type = INSTR_ALU_SHL_HH;
2930
2931                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2932                 instr->alu.dst.n_bits = fdst->n_bits;
2933                 instr->alu.dst.offset = fdst->offset / 8;
2934                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2935                 instr->alu.src.n_bits = fsrc->n_bits;
2936                 instr->alu.src.offset = fsrc->offset / 8;
2937                 return 0;
2938         }
2939
2940         /* SHL_MI, SHL_HI. */
2941         src_val = strtoull(src, &src, 0);
2942         CHECK(!src[0], EINVAL);
2943
2944         instr->type = INSTR_ALU_SHL_MI;
2945         if (dst[0] == 'h')
2946                 instr->type = INSTR_ALU_SHL_HI;
2947
2948         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2949         instr->alu.dst.n_bits = fdst->n_bits;
2950         instr->alu.dst.offset = fdst->offset / 8;
2951         instr->alu.src_val = src_val;
2952         return 0;
2953 }
2954
2955 static int
2956 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2957                         struct action *action,
2958                         char **tokens,
2959                         int n_tokens,
2960                         struct instruction *instr,
2961                         struct instruction_data *data __rte_unused)
2962 {
2963         char *dst = tokens[1], *src = tokens[2];
2964         struct field *fdst, *fsrc;
2965         uint64_t src_val;
2966         uint32_t dst_struct_id = 0, src_struct_id = 0;
2967
2968         CHECK(n_tokens == 3, EINVAL);
2969
2970         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2971         CHECK(fdst, EINVAL);
2972         CHECK(!fdst->var_size, EINVAL);
2973
2974         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2975         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2976         if (fsrc) {
2977                 CHECK(!fsrc->var_size, EINVAL);
2978
2979                 instr->type = INSTR_ALU_SHR;
2980                 if (dst[0] == 'h' && src[0] != 'h')
2981                         instr->type = INSTR_ALU_SHR_HM;
2982                 if (dst[0] != 'h' && src[0] == 'h')
2983                         instr->type = INSTR_ALU_SHR_MH;
2984                 if (dst[0] == 'h' && src[0] == 'h')
2985                         instr->type = INSTR_ALU_SHR_HH;
2986
2987                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2988                 instr->alu.dst.n_bits = fdst->n_bits;
2989                 instr->alu.dst.offset = fdst->offset / 8;
2990                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2991                 instr->alu.src.n_bits = fsrc->n_bits;
2992                 instr->alu.src.offset = fsrc->offset / 8;
2993                 return 0;
2994         }
2995
2996         /* SHR_MI, SHR_HI. */
2997         src_val = strtoull(src, &src, 0);
2998         CHECK(!src[0], EINVAL);
2999
3000         instr->type = INSTR_ALU_SHR_MI;
3001         if (dst[0] == 'h')
3002                 instr->type = INSTR_ALU_SHR_HI;
3003
3004         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3005         instr->alu.dst.n_bits = fdst->n_bits;
3006         instr->alu.dst.offset = fdst->offset / 8;
3007         instr->alu.src_val = src_val;
3008         return 0;
3009 }
3010
3011 static int
3012 instr_alu_and_translate(struct rte_swx_pipeline *p,
3013                         struct action *action,
3014                         char **tokens,
3015                         int n_tokens,
3016                         struct instruction *instr,
3017                         struct instruction_data *data __rte_unused)
3018 {
3019         char *dst = tokens[1], *src = tokens[2];
3020         struct field *fdst, *fsrc;
3021         uint64_t src_val;
3022         uint32_t dst_struct_id = 0, src_struct_id = 0;
3023
3024         CHECK(n_tokens == 3, EINVAL);
3025
3026         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3027         CHECK(fdst, EINVAL);
3028         CHECK(!fdst->var_size, EINVAL);
3029
3030         /* AND, AND_MH, AND_HM, AND_HH. */
3031         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3032         if (fsrc) {
3033                 CHECK(!fsrc->var_size, EINVAL);
3034
3035                 instr->type = INSTR_ALU_AND;
3036                 if (dst[0] != 'h' && src[0] == 'h')
3037                         instr->type = INSTR_ALU_AND_MH;
3038                 if (dst[0] == 'h' && src[0] != 'h')
3039                         instr->type = INSTR_ALU_AND_HM;
3040                 if (dst[0] == 'h' && src[0] == 'h')
3041                         instr->type = INSTR_ALU_AND_HH;
3042
3043                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3044                 instr->alu.dst.n_bits = fdst->n_bits;
3045                 instr->alu.dst.offset = fdst->offset / 8;
3046                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3047                 instr->alu.src.n_bits = fsrc->n_bits;
3048                 instr->alu.src.offset = fsrc->offset / 8;
3049                 return 0;
3050         }
3051
3052         /* AND_I. */
3053         src_val = strtoull(src, &src, 0);
3054         CHECK(!src[0], EINVAL);
3055
3056         if (dst[0] == 'h')
3057                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3058
3059         instr->type = INSTR_ALU_AND_I;
3060         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3061         instr->alu.dst.n_bits = fdst->n_bits;
3062         instr->alu.dst.offset = fdst->offset / 8;
3063         instr->alu.src_val = src_val;
3064         return 0;
3065 }
3066
3067 static int
3068 instr_alu_or_translate(struct rte_swx_pipeline *p,
3069                        struct action *action,
3070                        char **tokens,
3071                        int n_tokens,
3072                        struct instruction *instr,
3073                        struct instruction_data *data __rte_unused)
3074 {
3075         char *dst = tokens[1], *src = tokens[2];
3076         struct field *fdst, *fsrc;
3077         uint64_t src_val;
3078         uint32_t dst_struct_id = 0, src_struct_id = 0;
3079
3080         CHECK(n_tokens == 3, EINVAL);
3081
3082         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3083         CHECK(fdst, EINVAL);
3084         CHECK(!fdst->var_size, EINVAL);
3085
3086         /* OR, OR_MH, OR_HM, OR_HH. */
3087         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3088         if (fsrc) {
3089                 CHECK(!fsrc->var_size, EINVAL);
3090
3091                 instr->type = INSTR_ALU_OR;
3092                 if (dst[0] != 'h' && src[0] == 'h')
3093                         instr->type = INSTR_ALU_OR_MH;
3094                 if (dst[0] == 'h' && src[0] != 'h')
3095                         instr->type = INSTR_ALU_OR_HM;
3096                 if (dst[0] == 'h' && src[0] == 'h')
3097                         instr->type = INSTR_ALU_OR_HH;
3098
3099                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3100                 instr->alu.dst.n_bits = fdst->n_bits;
3101                 instr->alu.dst.offset = fdst->offset / 8;
3102                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3103                 instr->alu.src.n_bits = fsrc->n_bits;
3104                 instr->alu.src.offset = fsrc->offset / 8;
3105                 return 0;
3106         }
3107
3108         /* OR_I. */
3109         src_val = strtoull(src, &src, 0);
3110         CHECK(!src[0], EINVAL);
3111
3112         if (dst[0] == 'h')
3113                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3114
3115         instr->type = INSTR_ALU_OR_I;
3116         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3117         instr->alu.dst.n_bits = fdst->n_bits;
3118         instr->alu.dst.offset = fdst->offset / 8;
3119         instr->alu.src_val = src_val;
3120         return 0;
3121 }
3122
3123 static int
3124 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3125                         struct action *action,
3126                         char **tokens,
3127                         int n_tokens,
3128                         struct instruction *instr,
3129                         struct instruction_data *data __rte_unused)
3130 {
3131         char *dst = tokens[1], *src = tokens[2];
3132         struct field *fdst, *fsrc;
3133         uint64_t src_val;
3134         uint32_t dst_struct_id = 0, src_struct_id = 0;
3135
3136         CHECK(n_tokens == 3, EINVAL);
3137
3138         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3139         CHECK(fdst, EINVAL);
3140         CHECK(!fdst->var_size, EINVAL);
3141
3142         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3143         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3144         if (fsrc) {
3145                 CHECK(!fsrc->var_size, EINVAL);
3146
3147                 instr->type = INSTR_ALU_XOR;
3148                 if (dst[0] != 'h' && src[0] == 'h')
3149                         instr->type = INSTR_ALU_XOR_MH;
3150                 if (dst[0] == 'h' && src[0] != 'h')
3151                         instr->type = INSTR_ALU_XOR_HM;
3152                 if (dst[0] == 'h' && src[0] == 'h')
3153                         instr->type = INSTR_ALU_XOR_HH;
3154
3155                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3156                 instr->alu.dst.n_bits = fdst->n_bits;
3157                 instr->alu.dst.offset = fdst->offset / 8;
3158                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3159                 instr->alu.src.n_bits = fsrc->n_bits;
3160                 instr->alu.src.offset = fsrc->offset / 8;
3161                 return 0;
3162         }
3163
3164         /* XOR_I. */
3165         src_val = strtoull(src, &src, 0);
3166         CHECK(!src[0], EINVAL);
3167
3168         if (dst[0] == 'h')
3169                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3170
3171         instr->type = INSTR_ALU_XOR_I;
3172         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3173         instr->alu.dst.n_bits = fdst->n_bits;
3174         instr->alu.dst.offset = fdst->offset / 8;
3175         instr->alu.src_val = src_val;
3176         return 0;
3177 }
3178
3179 static inline void
3180 instr_alu_add_exec(struct rte_swx_pipeline *p)
3181 {
3182         struct thread *t = &p->threads[p->thread_id];
3183         struct instruction *ip = t->ip;
3184
3185         /* Structs */
3186         __instr_alu_add_exec(p, t, ip);
3187
3188         /* Thread. */
3189         thread_ip_inc(p);
3190 }
3191
3192 static inline void
3193 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3194 {
3195         struct thread *t = &p->threads[p->thread_id];
3196         struct instruction *ip = t->ip;
3197
3198         /* Structs. */
3199         __instr_alu_add_mh_exec(p, t, ip);
3200
3201         /* Thread. */
3202         thread_ip_inc(p);
3203 }
3204
3205 static inline void
3206 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3207 {
3208         struct thread *t = &p->threads[p->thread_id];
3209         struct instruction *ip = t->ip;
3210
3211         /* Structs. */
3212         __instr_alu_add_hm_exec(p, t, ip);
3213
3214         /* Thread. */
3215         thread_ip_inc(p);
3216 }
3217
3218 static inline void
3219 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3220 {
3221         struct thread *t = &p->threads[p->thread_id];
3222         struct instruction *ip = t->ip;
3223
3224         /* Structs. */
3225         __instr_alu_add_hh_exec(p, t, ip);
3226
3227         /* Thread. */
3228         thread_ip_inc(p);
3229 }
3230
3231 static inline void
3232 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3233 {
3234         struct thread *t = &p->threads[p->thread_id];
3235         struct instruction *ip = t->ip;
3236
3237         /* Structs. */
3238         __instr_alu_add_mi_exec(p, t, ip);
3239
3240         /* Thread. */
3241         thread_ip_inc(p);
3242 }
3243
3244 static inline void
3245 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3246 {
3247         struct thread *t = &p->threads[p->thread_id];
3248         struct instruction *ip = t->ip;
3249
3250         /* Structs. */
3251         __instr_alu_add_hi_exec(p, t, ip);
3252
3253         /* Thread. */
3254         thread_ip_inc(p);
3255 }
3256
3257 static inline void
3258 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3259 {
3260         struct thread *t = &p->threads[p->thread_id];
3261         struct instruction *ip = t->ip;
3262
3263         /* Structs. */
3264         __instr_alu_sub_exec(p, t, ip);
3265
3266         /* Thread. */
3267         thread_ip_inc(p);
3268 }
3269
3270 static inline void
3271 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3272 {
3273         struct thread *t = &p->threads[p->thread_id];
3274         struct instruction *ip = t->ip;
3275
3276         /* Structs. */
3277         __instr_alu_sub_mh_exec(p, t, ip);
3278
3279         /* Thread. */
3280         thread_ip_inc(p);
3281 }
3282
3283 static inline void
3284 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3285 {
3286         struct thread *t = &p->threads[p->thread_id];
3287         struct instruction *ip = t->ip;
3288
3289         /* Structs. */
3290         __instr_alu_sub_hm_exec(p, t, ip);
3291
3292         /* Thread. */
3293         thread_ip_inc(p);
3294 }
3295
3296 static inline void
3297 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3298 {
3299         struct thread *t = &p->threads[p->thread_id];
3300         struct instruction *ip = t->ip;
3301
3302         /* Structs. */
3303         __instr_alu_sub_hh_exec(p, t, ip);
3304
3305         /* Thread. */
3306         thread_ip_inc(p);
3307 }
3308
3309 static inline void
3310 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3311 {
3312         struct thread *t = &p->threads[p->thread_id];
3313         struct instruction *ip = t->ip;
3314
3315         /* Structs. */
3316         __instr_alu_sub_mi_exec(p, t, ip);
3317
3318         /* Thread. */
3319         thread_ip_inc(p);
3320 }
3321
3322 static inline void
3323 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3324 {
3325         struct thread *t = &p->threads[p->thread_id];
3326         struct instruction *ip = t->ip;
3327
3328         /* Structs. */
3329         __instr_alu_sub_hi_exec(p, t, ip);
3330
3331         /* Thread. */
3332         thread_ip_inc(p);
3333 }
3334
3335 static inline void
3336 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3337 {
3338         struct thread *t = &p->threads[p->thread_id];
3339         struct instruction *ip = t->ip;
3340
3341         /* Structs. */
3342         __instr_alu_shl_exec(p, t, ip);
3343
3344         /* Thread. */
3345         thread_ip_inc(p);
3346 }
3347
3348 static inline void
3349 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3350 {
3351         struct thread *t = &p->threads[p->thread_id];
3352         struct instruction *ip = t->ip;
3353
3354         /* Structs. */
3355         __instr_alu_shl_mh_exec(p, t, ip);
3356
3357         /* Thread. */
3358         thread_ip_inc(p);
3359 }
3360
3361 static inline void
3362 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3363 {
3364         struct thread *t = &p->threads[p->thread_id];
3365         struct instruction *ip = t->ip;
3366
3367         /* Structs. */
3368         __instr_alu_shl_hm_exec(p, t, ip);
3369
3370         /* Thread. */
3371         thread_ip_inc(p);
3372 }
3373
3374 static inline void
3375 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3376 {
3377         struct thread *t = &p->threads[p->thread_id];
3378         struct instruction *ip = t->ip;
3379
3380         /* Structs. */
3381         __instr_alu_shl_hh_exec(p, t, ip);
3382
3383         /* Thread. */
3384         thread_ip_inc(p);
3385 }
3386
3387 static inline void
3388 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3389 {
3390         struct thread *t = &p->threads[p->thread_id];
3391         struct instruction *ip = t->ip;
3392
3393         /* Structs. */
3394         __instr_alu_shl_mi_exec(p, t, ip);
3395
3396         /* Thread. */
3397         thread_ip_inc(p);
3398 }
3399
3400 static inline void
3401 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3402 {
3403         struct thread *t = &p->threads[p->thread_id];
3404         struct instruction *ip = t->ip;
3405
3406         /* Structs. */
3407         __instr_alu_shl_hi_exec(p, t, ip);
3408
3409         /* Thread. */
3410         thread_ip_inc(p);
3411 }
3412
3413 static inline void
3414 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3415 {
3416         struct thread *t = &p->threads[p->thread_id];
3417         struct instruction *ip = t->ip;
3418
3419         /* Structs. */
3420         __instr_alu_shr_exec(p, t, ip);
3421
3422         /* Thread. */
3423         thread_ip_inc(p);
3424 }
3425
3426 static inline void
3427 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3428 {
3429         struct thread *t = &p->threads[p->thread_id];
3430         struct instruction *ip = t->ip;
3431
3432         /* Structs. */
3433         __instr_alu_shr_mh_exec(p, t, ip);
3434
3435         /* Thread. */
3436         thread_ip_inc(p);
3437 }
3438
3439 static inline void
3440 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3441 {
3442         struct thread *t = &p->threads[p->thread_id];
3443         struct instruction *ip = t->ip;
3444
3445         /* Structs. */
3446         __instr_alu_shr_hm_exec(p, t, ip);
3447
3448         /* Thread. */
3449         thread_ip_inc(p);
3450 }
3451
3452 static inline void
3453 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3454 {
3455         struct thread *t = &p->threads[p->thread_id];
3456         struct instruction *ip = t->ip;
3457
3458         /* Structs. */
3459         __instr_alu_shr_hh_exec(p, t, ip);
3460
3461         /* Thread. */
3462         thread_ip_inc(p);
3463 }
3464
3465 static inline void
3466 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3467 {
3468         struct thread *t = &p->threads[p->thread_id];
3469         struct instruction *ip = t->ip;
3470
3471         /* Structs. */
3472         __instr_alu_shr_mi_exec(p, t, ip);
3473
3474         /* Thread. */
3475         thread_ip_inc(p);
3476 }
3477
3478 static inline void
3479 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3480 {
3481         struct thread *t = &p->threads[p->thread_id];
3482         struct instruction *ip = t->ip;
3483
3484         /* Structs. */
3485         __instr_alu_shr_hi_exec(p, t, ip);
3486
3487         /* Thread. */
3488         thread_ip_inc(p);
3489 }
3490
3491 static inline void
3492 instr_alu_and_exec(struct rte_swx_pipeline *p)
3493 {
3494         struct thread *t = &p->threads[p->thread_id];
3495         struct instruction *ip = t->ip;
3496
3497         /* Structs. */
3498         __instr_alu_and_exec(p, t, ip);
3499
3500         /* Thread. */
3501         thread_ip_inc(p);
3502 }
3503
3504 static inline void
3505 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3506 {
3507         struct thread *t = &p->threads[p->thread_id];
3508         struct instruction *ip = t->ip;
3509
3510         /* Structs. */
3511         __instr_alu_and_mh_exec(p, t, ip);
3512
3513         /* Thread. */
3514         thread_ip_inc(p);
3515 }
3516
3517 static inline void
3518 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3519 {
3520         struct thread *t = &p->threads[p->thread_id];
3521         struct instruction *ip = t->ip;
3522
3523         /* Structs. */
3524         __instr_alu_and_hm_exec(p, t, ip);
3525
3526         /* Thread. */
3527         thread_ip_inc(p);
3528 }
3529
3530 static inline void
3531 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3532 {
3533         struct thread *t = &p->threads[p->thread_id];
3534         struct instruction *ip = t->ip;
3535
3536         /* Structs. */
3537         __instr_alu_and_hh_exec(p, t, ip);
3538
3539         /* Thread. */
3540         thread_ip_inc(p);
3541 }
3542
3543 static inline void
3544 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3545 {
3546         struct thread *t = &p->threads[p->thread_id];
3547         struct instruction *ip = t->ip;
3548
3549         /* Structs. */
3550         __instr_alu_and_i_exec(p, t, ip);
3551
3552         /* Thread. */
3553         thread_ip_inc(p);
3554 }
3555
3556 static inline void
3557 instr_alu_or_exec(struct rte_swx_pipeline *p)
3558 {
3559         struct thread *t = &p->threads[p->thread_id];
3560         struct instruction *ip = t->ip;
3561
3562         /* Structs. */
3563         __instr_alu_or_exec(p, t, ip);
3564
3565         /* Thread. */
3566         thread_ip_inc(p);
3567 }
3568
3569 static inline void
3570 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3571 {
3572         struct thread *t = &p->threads[p->thread_id];
3573         struct instruction *ip = t->ip;
3574
3575         /* Structs. */
3576         __instr_alu_or_mh_exec(p, t, ip);
3577
3578         /* Thread. */
3579         thread_ip_inc(p);
3580 }
3581
3582 static inline void
3583 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3584 {
3585         struct thread *t = &p->threads[p->thread_id];
3586         struct instruction *ip = t->ip;
3587
3588         /* Structs. */
3589         __instr_alu_or_hm_exec(p, t, ip);
3590
3591         /* Thread. */
3592         thread_ip_inc(p);
3593 }
3594
3595 static inline void
3596 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3597 {
3598         struct thread *t = &p->threads[p->thread_id];
3599         struct instruction *ip = t->ip;
3600
3601         /* Structs. */
3602         __instr_alu_or_hh_exec(p, t, ip);
3603
3604         /* Thread. */
3605         thread_ip_inc(p);
3606 }
3607
3608 static inline void
3609 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3610 {
3611         struct thread *t = &p->threads[p->thread_id];
3612         struct instruction *ip = t->ip;
3613
3614         /* Structs. */
3615         __instr_alu_or_i_exec(p, t, ip);
3616
3617         /* Thread. */
3618         thread_ip_inc(p);
3619 }
3620
3621 static inline void
3622 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3623 {
3624         struct thread *t = &p->threads[p->thread_id];
3625         struct instruction *ip = t->ip;
3626
3627         /* Structs. */
3628         __instr_alu_xor_exec(p, t, ip);
3629
3630         /* Thread. */
3631         thread_ip_inc(p);
3632 }
3633
3634 static inline void
3635 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3636 {
3637         struct thread *t = &p->threads[p->thread_id];
3638         struct instruction *ip = t->ip;
3639
3640         /* Structs. */
3641         __instr_alu_xor_mh_exec(p, t, ip);
3642
3643         /* Thread. */
3644         thread_ip_inc(p);
3645 }
3646
3647 static inline void
3648 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3649 {
3650         struct thread *t = &p->threads[p->thread_id];
3651         struct instruction *ip = t->ip;
3652
3653         /* Structs. */
3654         __instr_alu_xor_hm_exec(p, t, ip);
3655
3656         /* Thread. */
3657         thread_ip_inc(p);
3658 }
3659
3660 static inline void
3661 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3662 {
3663         struct thread *t = &p->threads[p->thread_id];
3664         struct instruction *ip = t->ip;
3665
3666         /* Structs. */
3667         __instr_alu_xor_hh_exec(p, t, ip);
3668
3669         /* Thread. */
3670         thread_ip_inc(p);
3671 }
3672
3673 static inline void
3674 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3675 {
3676         struct thread *t = &p->threads[p->thread_id];
3677         struct instruction *ip = t->ip;
3678
3679         /* Structs. */
3680         __instr_alu_xor_i_exec(p, t, ip);
3681
3682         /* Thread. */
3683         thread_ip_inc(p);
3684 }
3685
3686 static inline void
3687 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3688 {
3689         struct thread *t = &p->threads[p->thread_id];
3690         struct instruction *ip = t->ip;
3691
3692         /* Structs. */
3693         __instr_alu_ckadd_field_exec(p, t, ip);
3694
3695         /* Thread. */
3696         thread_ip_inc(p);
3697 }
3698
3699 static inline void
3700 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3701 {
3702         struct thread *t = &p->threads[p->thread_id];
3703         struct instruction *ip = t->ip;
3704
3705         /* Structs. */
3706         __instr_alu_cksub_field_exec(p, t, ip);
3707
3708         /* Thread. */
3709         thread_ip_inc(p);
3710 }
3711
3712 static inline void
3713 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3714 {
3715         struct thread *t = &p->threads[p->thread_id];
3716         struct instruction *ip = t->ip;
3717
3718         /* Structs. */
3719         __instr_alu_ckadd_struct20_exec(p, t, ip);
3720
3721         /* Thread. */
3722         thread_ip_inc(p);
3723 }
3724
3725 static inline void
3726 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3727 {
3728         struct thread *t = &p->threads[p->thread_id];
3729         struct instruction *ip = t->ip;
3730
3731         /* Structs. */
3732         __instr_alu_ckadd_struct_exec(p, t, ip);
3733
3734         /* Thread. */
3735         thread_ip_inc(p);
3736 }
3737
3738 /*
3739  * Register array.
3740  */
3741 static struct regarray *
3742 regarray_find(struct rte_swx_pipeline *p, const char *name);
3743
3744 static int
3745 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3746                       struct action *action,
3747                       char **tokens,
3748                       int n_tokens,
3749                       struct instruction *instr,
3750                       struct instruction_data *data __rte_unused)
3751 {
3752         char *regarray = tokens[1], *idx = tokens[2];
3753         struct regarray *r;
3754         struct field *fidx;
3755         uint32_t idx_struct_id, idx_val;
3756
3757         CHECK(n_tokens == 3, EINVAL);
3758
3759         r = regarray_find(p, regarray);
3760         CHECK(r, EINVAL);
3761
3762         /* REGPREFETCH_RH, REGPREFETCH_RM. */
3763         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3764         if (fidx) {
3765                 CHECK(!fidx->var_size, EINVAL);
3766
3767                 instr->type = INSTR_REGPREFETCH_RM;
3768                 if (idx[0] == 'h')
3769                         instr->type = INSTR_REGPREFETCH_RH;
3770
3771                 instr->regarray.regarray_id = r->id;
3772                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3773                 instr->regarray.idx.n_bits = fidx->n_bits;
3774                 instr->regarray.idx.offset = fidx->offset / 8;
3775                 instr->regarray.dstsrc_val = 0; /* Unused. */
3776                 return 0;
3777         }
3778
3779         /* REGPREFETCH_RI. */
3780         idx_val = strtoul(idx, &idx, 0);
3781         CHECK(!idx[0], EINVAL);
3782
3783         instr->type = INSTR_REGPREFETCH_RI;
3784         instr->regarray.regarray_id = r->id;
3785         instr->regarray.idx_val = idx_val;
3786         instr->regarray.dstsrc_val = 0; /* Unused. */
3787         return 0;
3788 }
3789
3790 static int
3791 instr_regrd_translate(struct rte_swx_pipeline *p,
3792                       struct action *action,
3793                       char **tokens,
3794                       int n_tokens,
3795                       struct instruction *instr,
3796                       struct instruction_data *data __rte_unused)
3797 {
3798         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3799         struct regarray *r;
3800         struct field *fdst, *fidx;
3801         uint32_t dst_struct_id, idx_struct_id, idx_val;
3802
3803         CHECK(n_tokens == 4, EINVAL);
3804
3805         r = regarray_find(p, regarray);
3806         CHECK(r, EINVAL);
3807
3808         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3809         CHECK(fdst, EINVAL);
3810         CHECK(!fdst->var_size, EINVAL);
3811
3812         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3813         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3814         if (fidx) {
3815                 CHECK(!fidx->var_size, EINVAL);
3816
3817                 instr->type = INSTR_REGRD_MRM;
3818                 if (dst[0] == 'h' && idx[0] != 'h')
3819                         instr->type = INSTR_REGRD_HRM;
3820                 if (dst[0] != 'h' && idx[0] == 'h')
3821                         instr->type = INSTR_REGRD_MRH;
3822                 if (dst[0] == 'h' && idx[0] == 'h')
3823                         instr->type = INSTR_REGRD_HRH;
3824
3825                 instr->regarray.regarray_id = r->id;
3826                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3827                 instr->regarray.idx.n_bits = fidx->n_bits;
3828                 instr->regarray.idx.offset = fidx->offset / 8;
3829                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3830                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3831                 instr->regarray.dstsrc.offset = fdst->offset / 8;
3832                 return 0;
3833         }
3834
3835         /* REGRD_MRI, REGRD_HRI. */
3836         idx_val = strtoul(idx, &idx, 0);
3837         CHECK(!idx[0], EINVAL);
3838
3839         instr->type = INSTR_REGRD_MRI;
3840         if (dst[0] == 'h')
3841                 instr->type = INSTR_REGRD_HRI;
3842
3843         instr->regarray.regarray_id = r->id;
3844         instr->regarray.idx_val = idx_val;
3845         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3846         instr->regarray.dstsrc.n_bits = fdst->n_bits;
3847         instr->regarray.dstsrc.offset = fdst->offset / 8;
3848         return 0;
3849 }
3850
3851 static int
3852 instr_regwr_translate(struct rte_swx_pipeline *p,
3853                       struct action *action,
3854                       char **tokens,
3855                       int n_tokens,
3856                       struct instruction *instr,
3857                       struct instruction_data *data __rte_unused)
3858 {
3859         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3860         struct regarray *r;
3861         struct field *fidx, *fsrc;
3862         uint64_t src_val;
3863         uint32_t idx_struct_id, idx_val, src_struct_id;
3864
3865         CHECK(n_tokens == 4, EINVAL);
3866
3867         r = regarray_find(p, regarray);
3868         CHECK(r, EINVAL);
3869
3870         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3871         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3872         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3873         if (fidx && fsrc) {
3874                 CHECK(!fidx->var_size, EINVAL);
3875                 CHECK(!fsrc->var_size, EINVAL);
3876
3877                 instr->type = INSTR_REGWR_RMM;
3878                 if (idx[0] == 'h' && src[0] != 'h')
3879                         instr->type = INSTR_REGWR_RHM;
3880                 if (idx[0] != 'h' && src[0] == 'h')
3881                         instr->type = INSTR_REGWR_RMH;
3882                 if (idx[0] == 'h' && src[0] == 'h')
3883                         instr->type = INSTR_REGWR_RHH;
3884
3885                 instr->regarray.regarray_id = r->id;
3886                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3887                 instr->regarray.idx.n_bits = fidx->n_bits;
3888                 instr->regarray.idx.offset = fidx->offset / 8;
3889                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3890                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3891                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3892                 return 0;
3893         }
3894
3895         /* REGWR_RHI, REGWR_RMI. */
3896         if (fidx && !fsrc) {
3897                 CHECK(!fidx->var_size, EINVAL);
3898
3899                 src_val = strtoull(src, &src, 0);
3900                 CHECK(!src[0], EINVAL);
3901
3902                 instr->type = INSTR_REGWR_RMI;
3903                 if (idx[0] == 'h')
3904                         instr->type = INSTR_REGWR_RHI;
3905
3906                 instr->regarray.regarray_id = r->id;
3907                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3908                 instr->regarray.idx.n_bits = fidx->n_bits;
3909                 instr->regarray.idx.offset = fidx->offset / 8;
3910                 instr->regarray.dstsrc_val = src_val;
3911                 return 0;
3912         }
3913
3914         /* REGWR_RIH, REGWR_RIM. */
3915         if (!fidx && fsrc) {
3916                 idx_val = strtoul(idx, &idx, 0);
3917                 CHECK(!idx[0], EINVAL);
3918
3919                 CHECK(!fsrc->var_size, EINVAL);
3920
3921                 instr->type = INSTR_REGWR_RIM;
3922                 if (src[0] == 'h')
3923                         instr->type = INSTR_REGWR_RIH;
3924
3925                 instr->regarray.regarray_id = r->id;
3926                 instr->regarray.idx_val = idx_val;
3927                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3928                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3929                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3930                 return 0;
3931         }
3932
3933         /* REGWR_RII. */
3934         src_val = strtoull(src, &src, 0);
3935         CHECK(!src[0], EINVAL);
3936
3937         idx_val = strtoul(idx, &idx, 0);
3938         CHECK(!idx[0], EINVAL);
3939
3940         instr->type = INSTR_REGWR_RII;
3941         instr->regarray.idx_val = idx_val;
3942         instr->regarray.dstsrc_val = src_val;
3943
3944         return 0;
3945 }
3946
3947 static int
3948 instr_regadd_translate(struct rte_swx_pipeline *p,
3949                        struct action *action,
3950                        char **tokens,
3951                        int n_tokens,
3952                        struct instruction *instr,
3953                        struct instruction_data *data __rte_unused)
3954 {
3955         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3956         struct regarray *r;
3957         struct field *fidx, *fsrc;
3958         uint64_t src_val;
3959         uint32_t idx_struct_id, idx_val, src_struct_id;
3960
3961         CHECK(n_tokens == 4, EINVAL);
3962
3963         r = regarray_find(p, regarray);
3964         CHECK(r, EINVAL);
3965
3966         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
3967         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3968         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3969         if (fidx && fsrc) {
3970                 CHECK(!fidx->var_size, EINVAL);
3971                 CHECK(!fsrc->var_size, EINVAL);
3972
3973                 instr->type = INSTR_REGADD_RMM;
3974                 if (idx[0] == 'h' && src[0] != 'h')
3975                         instr->type = INSTR_REGADD_RHM;
3976                 if (idx[0] != 'h' && src[0] == 'h')
3977                         instr->type = INSTR_REGADD_RMH;
3978                 if (idx[0] == 'h' && src[0] == 'h')
3979                         instr->type = INSTR_REGADD_RHH;
3980
3981                 instr->regarray.regarray_id = r->id;
3982                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3983                 instr->regarray.idx.n_bits = fidx->n_bits;
3984                 instr->regarray.idx.offset = fidx->offset / 8;
3985                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3986                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3987                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3988                 return 0;
3989         }
3990
3991         /* REGADD_RHI, REGADD_RMI. */
3992         if (fidx && !fsrc) {
3993                 CHECK(!fidx->var_size, EINVAL);
3994
3995                 src_val = strtoull(src, &src, 0);
3996                 CHECK(!src[0], EINVAL);
3997
3998                 instr->type = INSTR_REGADD_RMI;
3999                 if (idx[0] == 'h')
4000                         instr->type = INSTR_REGADD_RHI;
4001
4002                 instr->regarray.regarray_id = r->id;
4003                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4004                 instr->regarray.idx.n_bits = fidx->n_bits;
4005                 instr->regarray.idx.offset = fidx->offset / 8;
4006                 instr->regarray.dstsrc_val = src_val;
4007                 return 0;
4008         }
4009
4010         /* REGADD_RIH, REGADD_RIM. */
4011         if (!fidx && fsrc) {
4012                 idx_val = strtoul(idx, &idx, 0);
4013                 CHECK(!idx[0], EINVAL);
4014
4015                 CHECK(!fsrc->var_size, EINVAL);
4016
4017                 instr->type = INSTR_REGADD_RIM;
4018                 if (src[0] == 'h')
4019                         instr->type = INSTR_REGADD_RIH;
4020
4021                 instr->regarray.regarray_id = r->id;
4022                 instr->regarray.idx_val = idx_val;
4023                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4024                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4025                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4026                 return 0;
4027         }
4028
4029         /* REGADD_RII. */
4030         src_val = strtoull(src, &src, 0);
4031         CHECK(!src[0], EINVAL);
4032
4033         idx_val = strtoul(idx, &idx, 0);
4034         CHECK(!idx[0], EINVAL);
4035
4036         instr->type = INSTR_REGADD_RII;
4037         instr->regarray.idx_val = idx_val;
4038         instr->regarray.dstsrc_val = src_val;
4039         return 0;
4040 }
4041
4042 static inline void
4043 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4044 {
4045         struct thread *t = &p->threads[p->thread_id];
4046         struct instruction *ip = t->ip;
4047
4048         /* Structs. */
4049         __instr_regprefetch_rh_exec(p, t, ip);
4050
4051         /* Thread. */
4052         thread_ip_inc(p);
4053 }
4054
4055 static inline void
4056 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4057 {
4058         struct thread *t = &p->threads[p->thread_id];
4059         struct instruction *ip = t->ip;
4060
4061         /* Structs. */
4062         __instr_regprefetch_rm_exec(p, t, ip);
4063
4064         /* Thread. */
4065         thread_ip_inc(p);
4066 }
4067
4068 static inline void
4069 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4070 {
4071         struct thread *t = &p->threads[p->thread_id];
4072         struct instruction *ip = t->ip;
4073
4074         /* Structs. */
4075         __instr_regprefetch_ri_exec(p, t, ip);
4076
4077         /* Thread. */
4078         thread_ip_inc(p);
4079 }
4080
4081 static inline void
4082 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4083 {
4084         struct thread *t = &p->threads[p->thread_id];
4085         struct instruction *ip = t->ip;
4086
4087         /* Structs. */
4088         __instr_regrd_hrh_exec(p, t, ip);
4089
4090         /* Thread. */
4091         thread_ip_inc(p);
4092 }
4093
4094 static inline void
4095 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4096 {
4097         struct thread *t = &p->threads[p->thread_id];
4098         struct instruction *ip = t->ip;
4099
4100         /* Structs. */
4101         __instr_regrd_hrm_exec(p, t, ip);
4102
4103         /* Thread. */
4104         thread_ip_inc(p);
4105 }
4106
4107 static inline void
4108 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4109 {
4110         struct thread *t = &p->threads[p->thread_id];
4111         struct instruction *ip = t->ip;
4112
4113         /* Structs. */
4114         __instr_regrd_mrh_exec(p, t, ip);
4115
4116         /* Thread. */
4117         thread_ip_inc(p);
4118 }
4119
4120 static inline void
4121 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4122 {
4123         struct thread *t = &p->threads[p->thread_id];
4124         struct instruction *ip = t->ip;
4125
4126         /* Structs. */
4127         __instr_regrd_mrm_exec(p, t, ip);
4128
4129         /* Thread. */
4130         thread_ip_inc(p);
4131 }
4132
4133 static inline void
4134 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4135 {
4136         struct thread *t = &p->threads[p->thread_id];
4137         struct instruction *ip = t->ip;
4138
4139         /* Structs. */
4140         __instr_regrd_hri_exec(p, t, ip);
4141
4142         /* Thread. */
4143         thread_ip_inc(p);
4144 }
4145
4146 static inline void
4147 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4148 {
4149         struct thread *t = &p->threads[p->thread_id];
4150         struct instruction *ip = t->ip;
4151
4152         /* Structs. */
4153         __instr_regrd_mri_exec(p, t, ip);
4154
4155         /* Thread. */
4156         thread_ip_inc(p);
4157 }
4158
4159 static inline void
4160 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4161 {
4162         struct thread *t = &p->threads[p->thread_id];
4163         struct instruction *ip = t->ip;
4164
4165         /* Structs. */
4166         __instr_regwr_rhh_exec(p, t, ip);
4167
4168         /* Thread. */
4169         thread_ip_inc(p);
4170 }
4171
4172 static inline void
4173 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4174 {
4175         struct thread *t = &p->threads[p->thread_id];
4176         struct instruction *ip = t->ip;
4177
4178         /* Structs. */
4179         __instr_regwr_rhm_exec(p, t, ip);
4180
4181         /* Thread. */
4182         thread_ip_inc(p);
4183 }
4184
4185 static inline void
4186 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4187 {
4188         struct thread *t = &p->threads[p->thread_id];
4189         struct instruction *ip = t->ip;
4190
4191         /* Structs. */
4192         __instr_regwr_rmh_exec(p, t, ip);
4193
4194         /* Thread. */
4195         thread_ip_inc(p);
4196 }
4197
4198 static inline void
4199 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4200 {
4201         struct thread *t = &p->threads[p->thread_id];
4202         struct instruction *ip = t->ip;
4203
4204         /* Structs. */
4205         __instr_regwr_rmm_exec(p, t, ip);
4206
4207         /* Thread. */
4208         thread_ip_inc(p);
4209 }
4210
4211 static inline void
4212 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4213 {
4214         struct thread *t = &p->threads[p->thread_id];
4215         struct instruction *ip = t->ip;
4216
4217         /* Structs. */
4218         __instr_regwr_rhi_exec(p, t, ip);
4219
4220         /* Thread. */
4221         thread_ip_inc(p);
4222 }
4223
4224 static inline void
4225 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4226 {
4227         struct thread *t = &p->threads[p->thread_id];
4228         struct instruction *ip = t->ip;
4229
4230         /* Structs. */
4231         __instr_regwr_rmi_exec(p, t, ip);
4232
4233         /* Thread. */
4234         thread_ip_inc(p);
4235 }
4236
4237 static inline void
4238 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4239 {
4240         struct thread *t = &p->threads[p->thread_id];
4241         struct instruction *ip = t->ip;
4242
4243         /* Structs. */
4244         __instr_regwr_rih_exec(p, t, ip);
4245
4246         /* Thread. */
4247         thread_ip_inc(p);
4248 }
4249
4250 static inline void
4251 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4252 {
4253         struct thread *t = &p->threads[p->thread_id];
4254         struct instruction *ip = t->ip;
4255
4256         /* Structs. */
4257         __instr_regwr_rim_exec(p, t, ip);
4258
4259         /* Thread. */
4260         thread_ip_inc(p);
4261 }
4262
4263 static inline void
4264 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4265 {
4266         struct thread *t = &p->threads[p->thread_id];
4267         struct instruction *ip = t->ip;
4268
4269         /* Structs. */
4270         __instr_regwr_rii_exec(p, t, ip);
4271
4272         /* Thread. */
4273         thread_ip_inc(p);
4274 }
4275
4276 static inline void
4277 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4278 {
4279         struct thread *t = &p->threads[p->thread_id];
4280         struct instruction *ip = t->ip;
4281
4282         /* Structs. */
4283         __instr_regadd_rhh_exec(p, t, ip);
4284
4285         /* Thread. */
4286         thread_ip_inc(p);
4287 }
4288
4289 static inline void
4290 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4291 {
4292         struct thread *t = &p->threads[p->thread_id];
4293         struct instruction *ip = t->ip;
4294
4295         /* Structs. */
4296         __instr_regadd_rhm_exec(p, t, ip);
4297
4298         /* Thread. */
4299         thread_ip_inc(p);
4300 }
4301
4302 static inline void
4303 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4304 {
4305         struct thread *t = &p->threads[p->thread_id];
4306         struct instruction *ip = t->ip;
4307
4308         /* Structs. */
4309         __instr_regadd_rmh_exec(p, t, ip);
4310
4311         /* Thread. */
4312         thread_ip_inc(p);
4313 }
4314
4315 static inline void
4316 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4317 {
4318         struct thread *t = &p->threads[p->thread_id];
4319         struct instruction *ip = t->ip;
4320
4321         /* Structs. */
4322         __instr_regadd_rmm_exec(p, t, ip);
4323
4324         /* Thread. */
4325         thread_ip_inc(p);
4326 }
4327
4328 static inline void
4329 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4330 {
4331         struct thread *t = &p->threads[p->thread_id];
4332         struct instruction *ip = t->ip;
4333
4334         /* Structs. */
4335         __instr_regadd_rhi_exec(p, t, ip);
4336
4337         /* Thread. */
4338         thread_ip_inc(p);
4339 }
4340
4341 static inline void
4342 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4343 {
4344         struct thread *t = &p->threads[p->thread_id];
4345         struct instruction *ip = t->ip;
4346
4347         /* Structs. */
4348         __instr_regadd_rmi_exec(p, t, ip);
4349
4350         /* Thread. */
4351         thread_ip_inc(p);
4352 }
4353
4354 static inline void
4355 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4356 {
4357         struct thread *t = &p->threads[p->thread_id];
4358         struct instruction *ip = t->ip;
4359
4360         /* Structs. */
4361         __instr_regadd_rih_exec(p, t, ip);
4362
4363         /* Thread. */
4364         thread_ip_inc(p);
4365 }
4366
4367 static inline void
4368 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4369 {
4370         struct thread *t = &p->threads[p->thread_id];
4371         struct instruction *ip = t->ip;
4372
4373         /* Structs. */
4374         __instr_regadd_rim_exec(p, t, ip);
4375
4376         /* Thread. */
4377         thread_ip_inc(p);
4378 }
4379
4380 static inline void
4381 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4382 {
4383         struct thread *t = &p->threads[p->thread_id];
4384         struct instruction *ip = t->ip;
4385
4386         /* Structs. */
4387         __instr_regadd_rii_exec(p, t, ip);
4388
4389         /* Thread. */
4390         thread_ip_inc(p);
4391 }
4392
4393 /*
4394  * metarray.
4395  */
4396 static struct metarray *
4397 metarray_find(struct rte_swx_pipeline *p, const char *name);
4398
4399 static int
4400 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4401                             struct action *action,
4402                             char **tokens,
4403                             int n_tokens,
4404                             struct instruction *instr,
4405                             struct instruction_data *data __rte_unused)
4406 {
4407         char *metarray = tokens[1], *idx = tokens[2];
4408         struct metarray *m;
4409         struct field *fidx;
4410         uint32_t idx_struct_id, idx_val;
4411
4412         CHECK(n_tokens == 3, EINVAL);
4413
4414         m = metarray_find(p, metarray);
4415         CHECK(m, EINVAL);
4416
4417         /* METPREFETCH_H, METPREFETCH_M. */
4418         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4419         if (fidx) {
4420                 CHECK(!fidx->var_size, EINVAL);
4421
4422                 instr->type = INSTR_METPREFETCH_M;
4423                 if (idx[0] == 'h')
4424                         instr->type = INSTR_METPREFETCH_H;
4425
4426                 instr->meter.metarray_id = m->id;
4427                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4428                 instr->meter.idx.n_bits = fidx->n_bits;
4429                 instr->meter.idx.offset = fidx->offset / 8;
4430                 return 0;
4431         }
4432
4433         /* METPREFETCH_I. */
4434         idx_val = strtoul(idx, &idx, 0);
4435         CHECK(!idx[0], EINVAL);
4436
4437         instr->type = INSTR_METPREFETCH_I;
4438         instr->meter.metarray_id = m->id;
4439         instr->meter.idx_val = idx_val;
4440         return 0;
4441 }
4442
4443 static int
4444 instr_meter_translate(struct rte_swx_pipeline *p,
4445                       struct action *action,
4446                       char **tokens,
4447                       int n_tokens,
4448                       struct instruction *instr,
4449                       struct instruction_data *data __rte_unused)
4450 {
4451         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4452         char *color_in = tokens[4], *color_out = tokens[5];
4453         struct metarray *m;
4454         struct field *fidx, *flength, *fcin, *fcout;
4455         uint32_t idx_struct_id, length_struct_id;
4456         uint32_t color_in_struct_id, color_out_struct_id;
4457
4458         CHECK(n_tokens == 6, EINVAL);
4459
4460         m = metarray_find(p, metarray);
4461         CHECK(m, EINVAL);
4462
4463         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4464
4465         flength = struct_field_parse(p, action, length, &length_struct_id);
4466         CHECK(flength, EINVAL);
4467         CHECK(!flength->var_size, EINVAL);
4468
4469         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4470
4471         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4472         CHECK(fcout, EINVAL);
4473         CHECK(!fcout->var_size, EINVAL);
4474
4475         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4476         if (fidx && fcin) {
4477                 CHECK(!fidx->var_size, EINVAL);
4478                 CHECK(!fcin->var_size, EINVAL);
4479
4480                 instr->type = INSTR_METER_MMM;
4481                 if (idx[0] == 'h' && length[0] == 'h')
4482                         instr->type = INSTR_METER_HHM;
4483                 if (idx[0] == 'h' && length[0] != 'h')
4484                         instr->type = INSTR_METER_HMM;
4485                 if (idx[0] != 'h' && length[0] == 'h')
4486                         instr->type = INSTR_METER_MHM;
4487
4488                 instr->meter.metarray_id = m->id;
4489
4490                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4491                 instr->meter.idx.n_bits = fidx->n_bits;
4492                 instr->meter.idx.offset = fidx->offset / 8;
4493
4494                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4495                 instr->meter.length.n_bits = flength->n_bits;
4496                 instr->meter.length.offset = flength->offset / 8;
4497
4498                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4499                 instr->meter.color_in.n_bits = fcin->n_bits;
4500                 instr->meter.color_in.offset = fcin->offset / 8;
4501
4502                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4503                 instr->meter.color_out.n_bits = fcout->n_bits;
4504                 instr->meter.color_out.offset = fcout->offset / 8;
4505
4506                 return 0;
4507         }
4508
4509         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4510         if (fidx && !fcin) {
4511                 uint32_t color_in_val;
4512
4513                 CHECK(!fidx->var_size, EINVAL);
4514
4515                 color_in_val = strtoul(color_in, &color_in, 0);
4516                 CHECK(!color_in[0], EINVAL);
4517
4518                 instr->type = INSTR_METER_MMI;
4519                 if (idx[0] == 'h' && length[0] == 'h')
4520                         instr->type = INSTR_METER_HHI;
4521                 if (idx[0] == 'h' && length[0] != 'h')
4522                         instr->type = INSTR_METER_HMI;
4523                 if (idx[0] != 'h' && length[0] == 'h')
4524                         instr->type = INSTR_METER_MHI;
4525
4526                 instr->meter.metarray_id = m->id;
4527
4528                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4529                 instr->meter.idx.n_bits = fidx->n_bits;
4530                 instr->meter.idx.offset = fidx->offset / 8;
4531
4532                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4533                 instr->meter.length.n_bits = flength->n_bits;
4534                 instr->meter.length.offset = flength->offset / 8;
4535
4536                 instr->meter.color_in_val = color_in_val;
4537
4538                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4539                 instr->meter.color_out.n_bits = fcout->n_bits;
4540                 instr->meter.color_out.offset = fcout->offset / 8;
4541
4542                 return 0;
4543         }
4544
4545         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4546         if (!fidx && fcin) {
4547                 uint32_t idx_val;
4548
4549                 idx_val = strtoul(idx, &idx, 0);
4550                 CHECK(!idx[0], EINVAL);
4551
4552                 CHECK(!fcin->var_size, EINVAL);
4553
4554                 instr->type = INSTR_METER_IMM;
4555                 if (length[0] == 'h')
4556                         instr->type = INSTR_METER_IHM;
4557
4558                 instr->meter.metarray_id = m->id;
4559
4560                 instr->meter.idx_val = idx_val;
4561
4562                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4563                 instr->meter.length.n_bits = flength->n_bits;
4564                 instr->meter.length.offset = flength->offset / 8;
4565
4566                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4567                 instr->meter.color_in.n_bits = fcin->n_bits;
4568                 instr->meter.color_in.offset = fcin->offset / 8;
4569
4570                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4571                 instr->meter.color_out.n_bits = fcout->n_bits;
4572                 instr->meter.color_out.offset = fcout->offset / 8;
4573
4574                 return 0;
4575         }
4576
4577         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4578         if (!fidx && !fcin) {
4579                 uint32_t idx_val, color_in_val;
4580
4581                 idx_val = strtoul(idx, &idx, 0);
4582                 CHECK(!idx[0], EINVAL);
4583
4584                 color_in_val = strtoul(color_in, &color_in, 0);
4585                 CHECK(!color_in[0], EINVAL);
4586
4587                 instr->type = INSTR_METER_IMI;
4588                 if (length[0] == 'h')
4589                         instr->type = INSTR_METER_IHI;
4590
4591                 instr->meter.metarray_id = m->id;
4592
4593                 instr->meter.idx_val = idx_val;
4594
4595                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4596                 instr->meter.length.n_bits = flength->n_bits;
4597                 instr->meter.length.offset = flength->offset / 8;
4598
4599                 instr->meter.color_in_val = color_in_val;
4600
4601                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4602                 instr->meter.color_out.n_bits = fcout->n_bits;
4603                 instr->meter.color_out.offset = fcout->offset / 8;
4604
4605                 return 0;
4606         }
4607
4608         CHECK(0, EINVAL);
4609 }
4610
4611 static inline void
4612 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4613 {
4614         struct thread *t = &p->threads[p->thread_id];
4615         struct instruction *ip = t->ip;
4616
4617         /* Structs. */
4618         __instr_metprefetch_h_exec(p, t, ip);
4619
4620         /* Thread. */
4621         thread_ip_inc(p);
4622 }
4623
4624 static inline void
4625 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4626 {
4627         struct thread *t = &p->threads[p->thread_id];
4628         struct instruction *ip = t->ip;
4629
4630         /* Structs. */
4631         __instr_metprefetch_m_exec(p, t, ip);
4632
4633         /* Thread. */
4634         thread_ip_inc(p);
4635 }
4636
4637 static inline void
4638 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4639 {
4640         struct thread *t = &p->threads[p->thread_id];
4641         struct instruction *ip = t->ip;
4642
4643         /* Structs. */
4644         __instr_metprefetch_i_exec(p, t, ip);
4645
4646         /* Thread. */
4647         thread_ip_inc(p);
4648 }
4649
4650 static inline void
4651 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4652 {
4653         struct thread *t = &p->threads[p->thread_id];
4654         struct instruction *ip = t->ip;
4655
4656         /* Structs. */
4657         __instr_meter_hhm_exec(p, t, ip);
4658
4659         /* Thread. */
4660         thread_ip_inc(p);
4661 }
4662
4663 static inline void
4664 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4665 {
4666         struct thread *t = &p->threads[p->thread_id];
4667         struct instruction *ip = t->ip;
4668
4669         /* Structs. */
4670         __instr_meter_hhi_exec(p, t, ip);
4671
4672         /* Thread. */
4673         thread_ip_inc(p);
4674 }
4675
4676 static inline void
4677 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4678 {
4679         struct thread *t = &p->threads[p->thread_id];
4680         struct instruction *ip = t->ip;
4681
4682         /* Structs. */
4683         __instr_meter_hmm_exec(p, t, ip);
4684
4685         /* Thread. */
4686         thread_ip_inc(p);
4687 }
4688
4689 static inline void
4690 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4691 {
4692         struct thread *t = &p->threads[p->thread_id];
4693         struct instruction *ip = t->ip;
4694
4695         /* Structs. */
4696         __instr_meter_hmi_exec(p, t, ip);
4697
4698         /* Thread. */
4699         thread_ip_inc(p);
4700 }
4701
4702 static inline void
4703 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4704 {
4705         struct thread *t = &p->threads[p->thread_id];
4706         struct instruction *ip = t->ip;
4707
4708         /* Structs. */
4709         __instr_meter_mhm_exec(p, t, ip);
4710
4711         /* Thread. */
4712         thread_ip_inc(p);
4713 }
4714
4715 static inline void
4716 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4717 {
4718         struct thread *t = &p->threads[p->thread_id];
4719         struct instruction *ip = t->ip;
4720
4721         /* Structs. */
4722         __instr_meter_mhi_exec(p, t, ip);
4723
4724         /* Thread. */
4725         thread_ip_inc(p);
4726 }
4727
4728 static inline void
4729 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4730 {
4731         struct thread *t = &p->threads[p->thread_id];
4732         struct instruction *ip = t->ip;
4733
4734         /* Structs. */
4735         __instr_meter_mmm_exec(p, t, ip);
4736
4737         /* Thread. */
4738         thread_ip_inc(p);
4739 }
4740
4741 static inline void
4742 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4743 {
4744         struct thread *t = &p->threads[p->thread_id];
4745         struct instruction *ip = t->ip;
4746
4747         /* Structs. */
4748         __instr_meter_mmi_exec(p, t, ip);
4749
4750         /* Thread. */
4751         thread_ip_inc(p);
4752 }
4753
4754 static inline void
4755 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4756 {
4757         struct thread *t = &p->threads[p->thread_id];
4758         struct instruction *ip = t->ip;
4759
4760         /* Structs. */
4761         __instr_meter_ihm_exec(p, t, ip);
4762
4763         /* Thread. */
4764         thread_ip_inc(p);
4765 }
4766
4767 static inline void
4768 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4769 {
4770         struct thread *t = &p->threads[p->thread_id];
4771         struct instruction *ip = t->ip;
4772
4773         /* Structs. */
4774         __instr_meter_ihi_exec(p, t, ip);
4775
4776         /* Thread. */
4777         thread_ip_inc(p);
4778 }
4779
4780 static inline void
4781 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4782 {
4783         struct thread *t = &p->threads[p->thread_id];
4784         struct instruction *ip = t->ip;
4785
4786         /* Structs. */
4787         __instr_meter_imm_exec(p, t, ip);
4788
4789         /* Thread. */
4790         thread_ip_inc(p);
4791 }
4792
4793 static inline void
4794 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4795 {
4796         struct thread *t = &p->threads[p->thread_id];
4797         struct instruction *ip = t->ip;
4798
4799         /* Structs. */
4800         __instr_meter_imi_exec(p, t, ip);
4801
4802         /* Thread. */
4803         thread_ip_inc(p);
4804 }
4805
4806 /*
4807  * jmp.
4808  */
4809 static int
4810 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4811                     struct action *action __rte_unused,
4812                     char **tokens,
4813                     int n_tokens,
4814                     struct instruction *instr,
4815                     struct instruction_data *data)
4816 {
4817         CHECK(n_tokens == 2, EINVAL);
4818
4819         strcpy(data->jmp_label, tokens[1]);
4820
4821         instr->type = INSTR_JMP;
4822         instr->jmp.ip = NULL; /* Resolved later. */
4823         return 0;
4824 }
4825
4826 static int
4827 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4828                           struct action *action __rte_unused,
4829                           char **tokens,
4830                           int n_tokens,
4831                           struct instruction *instr,
4832                           struct instruction_data *data)
4833 {
4834         struct header *h;
4835
4836         CHECK(n_tokens == 3, EINVAL);
4837
4838         strcpy(data->jmp_label, tokens[1]);
4839
4840         h = header_parse(p, tokens[2]);
4841         CHECK(h, EINVAL);
4842
4843         instr->type = INSTR_JMP_VALID;
4844         instr->jmp.ip = NULL; /* Resolved later. */
4845         instr->jmp.header_id = h->id;
4846         return 0;
4847 }
4848
4849 static int
4850 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4851                             struct action *action __rte_unused,
4852                             char **tokens,
4853                             int n_tokens,
4854                             struct instruction *instr,
4855                             struct instruction_data *data)
4856 {
4857         struct header *h;
4858
4859         CHECK(n_tokens == 3, EINVAL);
4860
4861         strcpy(data->jmp_label, tokens[1]);
4862
4863         h = header_parse(p, tokens[2]);
4864         CHECK(h, EINVAL);
4865
4866         instr->type = INSTR_JMP_INVALID;
4867         instr->jmp.ip = NULL; /* Resolved later. */
4868         instr->jmp.header_id = h->id;
4869         return 0;
4870 }
4871
4872 static int
4873 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4874                         struct action *action,
4875                         char **tokens,
4876                         int n_tokens,
4877                         struct instruction *instr,
4878                         struct instruction_data *data)
4879 {
4880         CHECK(!action, EINVAL);
4881         CHECK(n_tokens == 2, EINVAL);
4882
4883         strcpy(data->jmp_label, tokens[1]);
4884
4885         instr->type = INSTR_JMP_HIT;
4886         instr->jmp.ip = NULL; /* Resolved later. */
4887         return 0;
4888 }
4889
4890 static int
4891 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4892                          struct action *action,
4893                          char **tokens,
4894                          int n_tokens,
4895                          struct instruction *instr,
4896                          struct instruction_data *data)
4897 {
4898         CHECK(!action, EINVAL);
4899         CHECK(n_tokens == 2, EINVAL);
4900
4901         strcpy(data->jmp_label, tokens[1]);
4902
4903         instr->type = INSTR_JMP_MISS;
4904         instr->jmp.ip = NULL; /* Resolved later. */
4905         return 0;
4906 }
4907
4908 static int
4909 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4910                                struct action *action,
4911                                char **tokens,
4912                                int n_tokens,
4913                                struct instruction *instr,
4914                                struct instruction_data *data)
4915 {
4916         struct action *a;
4917
4918         CHECK(!action, EINVAL);
4919         CHECK(n_tokens == 3, EINVAL);
4920
4921         strcpy(data->jmp_label, tokens[1]);
4922
4923         a = action_find(p, tokens[2]);
4924         CHECK(a, EINVAL);
4925
4926         instr->type = INSTR_JMP_ACTION_HIT;
4927         instr->jmp.ip = NULL; /* Resolved later. */
4928         instr->jmp.action_id = a->id;
4929         return 0;
4930 }
4931
4932 static int
4933 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4934                                 struct action *action,
4935                                 char **tokens,
4936                                 int n_tokens,
4937                                 struct instruction *instr,
4938                                 struct instruction_data *data)
4939 {
4940         struct action *a;
4941
4942         CHECK(!action, EINVAL);
4943         CHECK(n_tokens == 3, EINVAL);
4944
4945         strcpy(data->jmp_label, tokens[1]);
4946
4947         a = action_find(p, tokens[2]);
4948         CHECK(a, EINVAL);
4949
4950         instr->type = INSTR_JMP_ACTION_MISS;
4951         instr->jmp.ip = NULL; /* Resolved later. */
4952         instr->jmp.action_id = a->id;
4953         return 0;
4954 }
4955
4956 static int
4957 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4958                        struct action *action,
4959                        char **tokens,
4960                        int n_tokens,
4961                        struct instruction *instr,
4962                        struct instruction_data *data)
4963 {
4964         char *a = tokens[2], *b = tokens[3];
4965         struct field *fa, *fb;
4966         uint64_t b_val;
4967         uint32_t a_struct_id, b_struct_id;
4968
4969         CHECK(n_tokens == 4, EINVAL);
4970
4971         strcpy(data->jmp_label, tokens[1]);
4972
4973         fa = struct_field_parse(p, action, a, &a_struct_id);
4974         CHECK(fa, EINVAL);
4975         CHECK(!fa->var_size, EINVAL);
4976
4977         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
4978         fb = struct_field_parse(p, action, b, &b_struct_id);
4979         if (fb) {
4980                 CHECK(!fb->var_size, EINVAL);
4981
4982                 instr->type = INSTR_JMP_EQ;
4983                 if (a[0] != 'h' && b[0] == 'h')
4984                         instr->type = INSTR_JMP_EQ_MH;
4985                 if (a[0] == 'h' && b[0] != 'h')
4986                         instr->type = INSTR_JMP_EQ_HM;
4987                 if (a[0] == 'h' && b[0] == 'h')
4988                         instr->type = INSTR_JMP_EQ_HH;
4989                 instr->jmp.ip = NULL; /* Resolved later. */
4990
4991                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4992                 instr->jmp.a.n_bits = fa->n_bits;
4993                 instr->jmp.a.offset = fa->offset / 8;
4994                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4995                 instr->jmp.b.n_bits = fb->n_bits;
4996                 instr->jmp.b.offset = fb->offset / 8;
4997                 return 0;
4998         }
4999
5000         /* JMP_EQ_I. */
5001         b_val = strtoull(b, &b, 0);
5002         CHECK(!b[0], EINVAL);
5003
5004         if (a[0] == 'h')
5005                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5006
5007         instr->type = INSTR_JMP_EQ_I;
5008         instr->jmp.ip = NULL; /* Resolved later. */
5009         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5010         instr->jmp.a.n_bits = fa->n_bits;
5011         instr->jmp.a.offset = fa->offset / 8;
5012         instr->jmp.b_val = b_val;
5013         return 0;
5014 }
5015
5016 static int
5017 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5018                         struct action *action,
5019                         char **tokens,
5020                         int n_tokens,
5021                         struct instruction *instr,
5022                         struct instruction_data *data)
5023 {
5024         char *a = tokens[2], *b = tokens[3];
5025         struct field *fa, *fb;
5026         uint64_t b_val;
5027         uint32_t a_struct_id, b_struct_id;
5028
5029         CHECK(n_tokens == 4, EINVAL);
5030
5031         strcpy(data->jmp_label, tokens[1]);
5032
5033         fa = struct_field_parse(p, action, a, &a_struct_id);
5034         CHECK(fa, EINVAL);
5035         CHECK(!fa->var_size, EINVAL);
5036
5037         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5038         fb = struct_field_parse(p, action, b, &b_struct_id);
5039         if (fb) {
5040                 CHECK(!fb->var_size, EINVAL);
5041
5042                 instr->type = INSTR_JMP_NEQ;
5043                 if (a[0] != 'h' && b[0] == 'h')
5044                         instr->type = INSTR_JMP_NEQ_MH;
5045                 if (a[0] == 'h' && b[0] != 'h')
5046                         instr->type = INSTR_JMP_NEQ_HM;
5047                 if (a[0] == 'h' && b[0] == 'h')
5048                         instr->type = INSTR_JMP_NEQ_HH;
5049                 instr->jmp.ip = NULL; /* Resolved later. */
5050
5051                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5052                 instr->jmp.a.n_bits = fa->n_bits;
5053                 instr->jmp.a.offset = fa->offset / 8;
5054                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5055                 instr->jmp.b.n_bits = fb->n_bits;
5056                 instr->jmp.b.offset = fb->offset / 8;
5057                 return 0;
5058         }
5059
5060         /* JMP_NEQ_I. */
5061         b_val = strtoull(b, &b, 0);
5062         CHECK(!b[0], EINVAL);
5063
5064         if (a[0] == 'h')
5065                 b_val = hton64(b_val) >> (64 - fa->n_bits);
5066
5067         instr->type = INSTR_JMP_NEQ_I;
5068         instr->jmp.ip = NULL; /* Resolved later. */
5069         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5070         instr->jmp.a.n_bits = fa->n_bits;
5071         instr->jmp.a.offset = fa->offset / 8;
5072         instr->jmp.b_val = b_val;
5073         return 0;
5074 }
5075
5076 static int
5077 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5078                        struct action *action,
5079                        char **tokens,
5080                        int n_tokens,
5081                        struct instruction *instr,
5082                        struct instruction_data *data)
5083 {
5084         char *a = tokens[2], *b = tokens[3];
5085         struct field *fa, *fb;
5086         uint64_t b_val;
5087         uint32_t a_struct_id, b_struct_id;
5088
5089         CHECK(n_tokens == 4, EINVAL);
5090
5091         strcpy(data->jmp_label, tokens[1]);
5092
5093         fa = struct_field_parse(p, action, a, &a_struct_id);
5094         CHECK(fa, EINVAL);
5095         CHECK(!fa->var_size, EINVAL);
5096
5097         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5098         fb = struct_field_parse(p, action, b, &b_struct_id);
5099         if (fb) {
5100                 CHECK(!fb->var_size, EINVAL);
5101
5102                 instr->type = INSTR_JMP_LT;
5103                 if (a[0] == 'h' && b[0] != 'h')
5104                         instr->type = INSTR_JMP_LT_HM;
5105                 if (a[0] != 'h' && b[0] == 'h')
5106                         instr->type = INSTR_JMP_LT_MH;
5107                 if (a[0] == 'h' && b[0] == 'h')
5108                         instr->type = INSTR_JMP_LT_HH;
5109                 instr->jmp.ip = NULL; /* Resolved later. */
5110
5111                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5112                 instr->jmp.a.n_bits = fa->n_bits;
5113                 instr->jmp.a.offset = fa->offset / 8;
5114                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5115                 instr->jmp.b.n_bits = fb->n_bits;
5116                 instr->jmp.b.offset = fb->offset / 8;
5117                 return 0;
5118         }
5119
5120         /* JMP_LT_MI, JMP_LT_HI. */
5121         b_val = strtoull(b, &b, 0);
5122         CHECK(!b[0], EINVAL);
5123
5124         instr->type = INSTR_JMP_LT_MI;
5125         if (a[0] == 'h')
5126                 instr->type = INSTR_JMP_LT_HI;
5127         instr->jmp.ip = NULL; /* Resolved later. */
5128
5129         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5130         instr->jmp.a.n_bits = fa->n_bits;
5131         instr->jmp.a.offset = fa->offset / 8;
5132         instr->jmp.b_val = b_val;
5133         return 0;
5134 }
5135
5136 static int
5137 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5138                        struct action *action,
5139                        char **tokens,
5140                        int n_tokens,
5141                        struct instruction *instr,
5142                        struct instruction_data *data)
5143 {
5144         char *a = tokens[2], *b = tokens[3];
5145         struct field *fa, *fb;
5146         uint64_t b_val;
5147         uint32_t a_struct_id, b_struct_id;
5148
5149         CHECK(n_tokens == 4, EINVAL);
5150
5151         strcpy(data->jmp_label, tokens[1]);
5152
5153         fa = struct_field_parse(p, action, a, &a_struct_id);
5154         CHECK(fa, EINVAL);
5155         CHECK(!fa->var_size, EINVAL);
5156
5157         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5158         fb = struct_field_parse(p, action, b, &b_struct_id);
5159         if (fb) {
5160                 CHECK(!fb->var_size, EINVAL);
5161
5162                 instr->type = INSTR_JMP_GT;
5163                 if (a[0] == 'h' && b[0] != 'h')
5164                         instr->type = INSTR_JMP_GT_HM;
5165                 if (a[0] != 'h' && b[0] == 'h')
5166                         instr->type = INSTR_JMP_GT_MH;
5167                 if (a[0] == 'h' && b[0] == 'h')
5168                         instr->type = INSTR_JMP_GT_HH;
5169                 instr->jmp.ip = NULL; /* Resolved later. */
5170
5171                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5172                 instr->jmp.a.n_bits = fa->n_bits;
5173                 instr->jmp.a.offset = fa->offset / 8;
5174                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5175                 instr->jmp.b.n_bits = fb->n_bits;
5176                 instr->jmp.b.offset = fb->offset / 8;
5177                 return 0;
5178         }
5179
5180         /* JMP_GT_MI, JMP_GT_HI. */
5181         b_val = strtoull(b, &b, 0);
5182         CHECK(!b[0], EINVAL);
5183
5184         instr->type = INSTR_JMP_GT_MI;
5185         if (a[0] == 'h')
5186                 instr->type = INSTR_JMP_GT_HI;
5187         instr->jmp.ip = NULL; /* Resolved later. */
5188
5189         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5190         instr->jmp.a.n_bits = fa->n_bits;
5191         instr->jmp.a.offset = fa->offset / 8;
5192         instr->jmp.b_val = b_val;
5193         return 0;
5194 }
5195
5196 static inline void
5197 instr_jmp_exec(struct rte_swx_pipeline *p)
5198 {
5199         struct thread *t = &p->threads[p->thread_id];
5200         struct instruction *ip = t->ip;
5201
5202         TRACE("[Thread %2u] jmp\n", p->thread_id);
5203
5204         thread_ip_set(t, ip->jmp.ip);
5205 }
5206
5207 static inline void
5208 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5209 {
5210         struct thread *t = &p->threads[p->thread_id];
5211         struct instruction *ip = t->ip;
5212         uint32_t header_id = ip->jmp.header_id;
5213
5214         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5215
5216         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5217 }
5218
5219 static inline void
5220 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5221 {
5222         struct thread *t = &p->threads[p->thread_id];
5223         struct instruction *ip = t->ip;
5224         uint32_t header_id = ip->jmp.header_id;
5225
5226         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5227
5228         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5229 }
5230
5231 static inline void
5232 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5233 {
5234         struct thread *t = &p->threads[p->thread_id];
5235         struct instruction *ip = t->ip;
5236         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5237
5238         TRACE("[Thread %2u] jmph\n", p->thread_id);
5239
5240         t->ip = ip_next[t->hit];
5241 }
5242
5243 static inline void
5244 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5245 {
5246         struct thread *t = &p->threads[p->thread_id];
5247         struct instruction *ip = t->ip;
5248         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5249
5250         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5251
5252         t->ip = ip_next[t->hit];
5253 }
5254
5255 static inline void
5256 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5257 {
5258         struct thread *t = &p->threads[p->thread_id];
5259         struct instruction *ip = t->ip;
5260
5261         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5262
5263         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5264 }
5265
5266 static inline void
5267 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5268 {
5269         struct thread *t = &p->threads[p->thread_id];
5270         struct instruction *ip = t->ip;
5271
5272         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5273
5274         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5275 }
5276
5277 static inline void
5278 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5279 {
5280         struct thread *t = &p->threads[p->thread_id];
5281         struct instruction *ip = t->ip;
5282
5283         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5284
5285         JMP_CMP(t, ip, ==);
5286 }
5287
5288 static inline void
5289 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5290 {
5291         struct thread *t = &p->threads[p->thread_id];
5292         struct instruction *ip = t->ip;
5293
5294         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5295
5296         JMP_CMP_MH(t, ip, ==);
5297 }
5298
5299 static inline void
5300 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5301 {
5302         struct thread *t = &p->threads[p->thread_id];
5303         struct instruction *ip = t->ip;
5304
5305         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5306
5307         JMP_CMP_HM(t, ip, ==);
5308 }
5309
5310 static inline void
5311 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5312 {
5313         struct thread *t = &p->threads[p->thread_id];
5314         struct instruction *ip = t->ip;
5315
5316         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5317
5318         JMP_CMP_HH_FAST(t, ip, ==);
5319 }
5320
5321 static inline void
5322 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5323 {
5324         struct thread *t = &p->threads[p->thread_id];
5325         struct instruction *ip = t->ip;
5326
5327         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5328
5329         JMP_CMP_I(t, ip, ==);
5330 }
5331
5332 static inline void
5333 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5334 {
5335         struct thread *t = &p->threads[p->thread_id];
5336         struct instruction *ip = t->ip;
5337
5338         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5339
5340         JMP_CMP(t, ip, !=);
5341 }
5342
5343 static inline void
5344 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5345 {
5346         struct thread *t = &p->threads[p->thread_id];
5347         struct instruction *ip = t->ip;
5348
5349         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5350
5351         JMP_CMP_MH(t, ip, !=);
5352 }
5353
5354 static inline void
5355 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5356 {
5357         struct thread *t = &p->threads[p->thread_id];
5358         struct instruction *ip = t->ip;
5359
5360         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5361
5362         JMP_CMP_HM(t, ip, !=);
5363 }
5364
5365 static inline void
5366 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5367 {
5368         struct thread *t = &p->threads[p->thread_id];
5369         struct instruction *ip = t->ip;
5370
5371         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5372
5373         JMP_CMP_HH_FAST(t, ip, !=);
5374 }
5375
5376 static inline void
5377 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5378 {
5379         struct thread *t = &p->threads[p->thread_id];
5380         struct instruction *ip = t->ip;
5381
5382         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5383
5384         JMP_CMP_I(t, ip, !=);
5385 }
5386
5387 static inline void
5388 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5389 {
5390         struct thread *t = &p->threads[p->thread_id];
5391         struct instruction *ip = t->ip;
5392
5393         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5394
5395         JMP_CMP(t, ip, <);
5396 }
5397
5398 static inline void
5399 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5400 {
5401         struct thread *t = &p->threads[p->thread_id];
5402         struct instruction *ip = t->ip;
5403
5404         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5405
5406         JMP_CMP_MH(t, ip, <);
5407 }
5408
5409 static inline void
5410 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5411 {
5412         struct thread *t = &p->threads[p->thread_id];
5413         struct instruction *ip = t->ip;
5414
5415         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5416
5417         JMP_CMP_HM(t, ip, <);
5418 }
5419
5420 static inline void
5421 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5422 {
5423         struct thread *t = &p->threads[p->thread_id];
5424         struct instruction *ip = t->ip;
5425
5426         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5427
5428         JMP_CMP_HH(t, ip, <);
5429 }
5430
5431 static inline void
5432 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5433 {
5434         struct thread *t = &p->threads[p->thread_id];
5435         struct instruction *ip = t->ip;
5436
5437         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5438
5439         JMP_CMP_MI(t, ip, <);
5440 }
5441
5442 static inline void
5443 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5444 {
5445         struct thread *t = &p->threads[p->thread_id];
5446         struct instruction *ip = t->ip;
5447
5448         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5449
5450         JMP_CMP_HI(t, ip, <);
5451 }
5452
5453 static inline void
5454 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5455 {
5456         struct thread *t = &p->threads[p->thread_id];
5457         struct instruction *ip = t->ip;
5458
5459         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5460
5461         JMP_CMP(t, ip, >);
5462 }
5463
5464 static inline void
5465 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5466 {
5467         struct thread *t = &p->threads[p->thread_id];
5468         struct instruction *ip = t->ip;
5469
5470         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5471
5472         JMP_CMP_MH(t, ip, >);
5473 }
5474
5475 static inline void
5476 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5477 {
5478         struct thread *t = &p->threads[p->thread_id];
5479         struct instruction *ip = t->ip;
5480
5481         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5482
5483         JMP_CMP_HM(t, ip, >);
5484 }
5485
5486 static inline void
5487 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5488 {
5489         struct thread *t = &p->threads[p->thread_id];
5490         struct instruction *ip = t->ip;
5491
5492         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5493
5494         JMP_CMP_HH(t, ip, >);
5495 }
5496
5497 static inline void
5498 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5499 {
5500         struct thread *t = &p->threads[p->thread_id];
5501         struct instruction *ip = t->ip;
5502
5503         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5504
5505         JMP_CMP_MI(t, ip, >);
5506 }
5507
5508 static inline void
5509 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5510 {
5511         struct thread *t = &p->threads[p->thread_id];
5512         struct instruction *ip = t->ip;
5513
5514         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5515
5516         JMP_CMP_HI(t, ip, >);
5517 }
5518
5519 /*
5520  * return.
5521  */
5522 static int
5523 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5524                        struct action *action,
5525                        char **tokens __rte_unused,
5526                        int n_tokens,
5527                        struct instruction *instr,
5528                        struct instruction_data *data __rte_unused)
5529 {
5530         CHECK(action, EINVAL);
5531         CHECK(n_tokens == 1, EINVAL);
5532
5533         instr->type = INSTR_RETURN;
5534         return 0;
5535 }
5536
5537 static inline void
5538 instr_return_exec(struct rte_swx_pipeline *p)
5539 {
5540         struct thread *t = &p->threads[p->thread_id];
5541
5542         TRACE("[Thread %2u] return\n", p->thread_id);
5543
5544         t->ip = t->ret;
5545 }
5546
5547 static int
5548 instr_translate(struct rte_swx_pipeline *p,
5549                 struct action *action,
5550                 char *string,
5551                 struct instruction *instr,
5552                 struct instruction_data *data)
5553 {
5554         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5555         int n_tokens = 0, tpos = 0;
5556
5557         /* Parse the instruction string into tokens. */
5558         for ( ; ; ) {
5559                 char *token;
5560
5561                 token = strtok_r(string, " \t\v", &string);
5562                 if (!token)
5563                         break;
5564
5565                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5566                 CHECK_NAME(token, EINVAL);
5567
5568                 tokens[n_tokens] = token;
5569                 n_tokens++;
5570         }
5571
5572         CHECK(n_tokens, EINVAL);
5573
5574         /* Handle the optional instruction label. */
5575         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5576                 strcpy(data->label, tokens[0]);
5577
5578                 tpos += 2;
5579                 CHECK(n_tokens - tpos, EINVAL);
5580         }
5581
5582         /* Identify the instruction type. */
5583         if (!strcmp(tokens[tpos], "rx"))
5584                 return instr_rx_translate(p,
5585                                           action,
5586                                           &tokens[tpos],
5587                                           n_tokens - tpos,
5588                                           instr,
5589                                           data);
5590
5591         if (!strcmp(tokens[tpos], "tx"))
5592                 return instr_tx_translate(p,
5593                                           action,
5594                                           &tokens[tpos],
5595                                           n_tokens - tpos,
5596                                           instr,
5597                                           data);
5598
5599         if (!strcmp(tokens[tpos], "drop"))
5600                 return instr_drop_translate(p,
5601                                             action,
5602                                             &tokens[tpos],
5603                                             n_tokens - tpos,
5604                                             instr,
5605                                             data);
5606
5607         if (!strcmp(tokens[tpos], "extract"))
5608                 return instr_hdr_extract_translate(p,
5609                                                    action,
5610                                                    &tokens[tpos],
5611                                                    n_tokens - tpos,
5612                                                    instr,
5613                                                    data);
5614
5615         if (!strcmp(tokens[tpos], "lookahead"))
5616                 return instr_hdr_lookahead_translate(p,
5617                                                      action,
5618                                                      &tokens[tpos],
5619                                                      n_tokens - tpos,
5620                                                      instr,
5621                                                      data);
5622
5623         if (!strcmp(tokens[tpos], "emit"))
5624                 return instr_hdr_emit_translate(p,
5625                                                 action,
5626                                                 &tokens[tpos],
5627                                                 n_tokens - tpos,
5628                                                 instr,
5629                                                 data);
5630
5631         if (!strcmp(tokens[tpos], "validate"))
5632                 return instr_hdr_validate_translate(p,
5633                                                     action,
5634                                                     &tokens[tpos],
5635                                                     n_tokens - tpos,
5636                                                     instr,
5637                                                     data);
5638
5639         if (!strcmp(tokens[tpos], "invalidate"))
5640                 return instr_hdr_invalidate_translate(p,
5641                                                       action,
5642                                                       &tokens[tpos],
5643                                                       n_tokens - tpos,
5644                                                       instr,
5645                                                       data);
5646
5647         if (!strcmp(tokens[tpos], "mov"))
5648                 return instr_mov_translate(p,
5649                                            action,
5650                                            &tokens[tpos],
5651                                            n_tokens - tpos,
5652                                            instr,
5653                                            data);
5654
5655         if (!strcmp(tokens[tpos], "add"))
5656                 return instr_alu_add_translate(p,
5657                                                action,
5658                                                &tokens[tpos],
5659                                                n_tokens - tpos,
5660                                                instr,
5661                                                data);
5662
5663         if (!strcmp(tokens[tpos], "sub"))
5664                 return instr_alu_sub_translate(p,
5665                                                action,
5666                                                &tokens[tpos],
5667                                                n_tokens - tpos,
5668                                                instr,
5669                                                data);
5670
5671         if (!strcmp(tokens[tpos], "ckadd"))
5672                 return instr_alu_ckadd_translate(p,
5673                                                  action,
5674                                                  &tokens[tpos],
5675                                                  n_tokens - tpos,
5676                                                  instr,
5677                                                  data);
5678
5679         if (!strcmp(tokens[tpos], "cksub"))
5680                 return instr_alu_cksub_translate(p,
5681                                                  action,
5682                                                  &tokens[tpos],
5683                                                  n_tokens - tpos,
5684                                                  instr,
5685                                                  data);
5686
5687         if (!strcmp(tokens[tpos], "and"))
5688                 return instr_alu_and_translate(p,
5689                                                action,
5690                                                &tokens[tpos],
5691                                                n_tokens - tpos,
5692                                                instr,
5693                                                data);
5694
5695         if (!strcmp(tokens[tpos], "or"))
5696                 return instr_alu_or_translate(p,
5697                                               action,
5698                                               &tokens[tpos],
5699                                               n_tokens - tpos,
5700                                               instr,
5701                                               data);
5702
5703         if (!strcmp(tokens[tpos], "xor"))
5704                 return instr_alu_xor_translate(p,
5705                                                action,
5706                                                &tokens[tpos],
5707                                                n_tokens - tpos,
5708                                                instr,
5709                                                data);
5710
5711         if (!strcmp(tokens[tpos], "shl"))
5712                 return instr_alu_shl_translate(p,
5713                                                action,
5714                                                &tokens[tpos],
5715                                                n_tokens - tpos,
5716                                                instr,
5717                                                data);
5718
5719         if (!strcmp(tokens[tpos], "shr"))
5720                 return instr_alu_shr_translate(p,
5721                                                action,
5722                                                &tokens[tpos],
5723                                                n_tokens - tpos,
5724                                                instr,
5725                                                data);
5726
5727         if (!strcmp(tokens[tpos], "regprefetch"))
5728                 return instr_regprefetch_translate(p,
5729                                                    action,
5730                                                    &tokens[tpos],
5731                                                    n_tokens - tpos,
5732                                                    instr,
5733                                                    data);
5734
5735         if (!strcmp(tokens[tpos], "regrd"))
5736                 return instr_regrd_translate(p,
5737                                              action,
5738                                              &tokens[tpos],
5739                                              n_tokens - tpos,
5740                                              instr,
5741                                              data);
5742
5743         if (!strcmp(tokens[tpos], "regwr"))
5744                 return instr_regwr_translate(p,
5745                                              action,
5746                                              &tokens[tpos],
5747                                              n_tokens - tpos,
5748                                              instr,
5749                                              data);
5750
5751         if (!strcmp(tokens[tpos], "regadd"))
5752                 return instr_regadd_translate(p,
5753                                               action,
5754                                               &tokens[tpos],
5755                                               n_tokens - tpos,
5756                                               instr,
5757                                               data);
5758
5759         if (!strcmp(tokens[tpos], "metprefetch"))
5760                 return instr_metprefetch_translate(p,
5761                                                    action,
5762                                                    &tokens[tpos],
5763                                                    n_tokens - tpos,
5764                                                    instr,
5765                                                    data);
5766
5767         if (!strcmp(tokens[tpos], "meter"))
5768                 return instr_meter_translate(p,
5769                                              action,
5770                                              &tokens[tpos],
5771                                              n_tokens - tpos,
5772                                              instr,
5773                                              data);
5774
5775         if (!strcmp(tokens[tpos], "table"))
5776                 return instr_table_translate(p,
5777                                              action,
5778                                              &tokens[tpos],
5779                                              n_tokens - tpos,
5780                                              instr,
5781                                              data);
5782
5783         if (!strcmp(tokens[tpos], "learn"))
5784                 return instr_learn_translate(p,
5785                                              action,
5786                                              &tokens[tpos],
5787                                              n_tokens - tpos,
5788                                              instr,
5789                                              data);
5790
5791         if (!strcmp(tokens[tpos], "forget"))
5792                 return instr_forget_translate(p,
5793                                               action,
5794                                               &tokens[tpos],
5795                                               n_tokens - tpos,
5796                                               instr,
5797                                               data);
5798
5799         if (!strcmp(tokens[tpos], "extern"))
5800                 return instr_extern_translate(p,
5801                                               action,
5802                                               &tokens[tpos],
5803                                               n_tokens - tpos,
5804                                               instr,
5805                                               data);
5806
5807         if (!strcmp(tokens[tpos], "jmp"))
5808                 return instr_jmp_translate(p,
5809                                            action,
5810                                            &tokens[tpos],
5811                                            n_tokens - tpos,
5812                                            instr,
5813                                            data);
5814
5815         if (!strcmp(tokens[tpos], "jmpv"))
5816                 return instr_jmp_valid_translate(p,
5817                                                  action,
5818                                                  &tokens[tpos],
5819                                                  n_tokens - tpos,
5820                                                  instr,
5821                                                  data);
5822
5823         if (!strcmp(tokens[tpos], "jmpnv"))
5824                 return instr_jmp_invalid_translate(p,
5825                                                    action,
5826                                                    &tokens[tpos],
5827                                                    n_tokens - tpos,
5828                                                    instr,
5829                                                    data);
5830
5831         if (!strcmp(tokens[tpos], "jmph"))
5832                 return instr_jmp_hit_translate(p,
5833                                                action,
5834                                                &tokens[tpos],
5835                                                n_tokens - tpos,
5836                                                instr,
5837                                                data);
5838
5839         if (!strcmp(tokens[tpos], "jmpnh"))
5840                 return instr_jmp_miss_translate(p,
5841                                                 action,
5842                                                 &tokens[tpos],
5843                                                 n_tokens - tpos,
5844                                                 instr,
5845                                                 data);
5846
5847         if (!strcmp(tokens[tpos], "jmpa"))
5848                 return instr_jmp_action_hit_translate(p,
5849                                                       action,
5850                                                       &tokens[tpos],
5851                                                       n_tokens - tpos,
5852                                                       instr,
5853                                                       data);
5854
5855         if (!strcmp(tokens[tpos], "jmpna"))
5856                 return instr_jmp_action_miss_translate(p,
5857                                                        action,
5858                                                        &tokens[tpos],
5859                                                        n_tokens - tpos,
5860                                                        instr,
5861                                                        data);
5862
5863         if (!strcmp(tokens[tpos], "jmpeq"))
5864                 return instr_jmp_eq_translate(p,
5865                                               action,
5866                                               &tokens[tpos],
5867                                               n_tokens - tpos,
5868                                               instr,
5869                                               data);
5870
5871         if (!strcmp(tokens[tpos], "jmpneq"))
5872                 return instr_jmp_neq_translate(p,
5873                                                action,
5874                                                &tokens[tpos],
5875                                                n_tokens - tpos,
5876                                                instr,
5877                                                data);
5878
5879         if (!strcmp(tokens[tpos], "jmplt"))
5880                 return instr_jmp_lt_translate(p,
5881                                               action,
5882                                               &tokens[tpos],
5883                                               n_tokens - tpos,
5884                                               instr,
5885                                               data);
5886
5887         if (!strcmp(tokens[tpos], "jmpgt"))
5888                 return instr_jmp_gt_translate(p,
5889                                               action,
5890                                               &tokens[tpos],
5891                                               n_tokens - tpos,
5892                                               instr,
5893                                               data);
5894
5895         if (!strcmp(tokens[tpos], "return"))
5896                 return instr_return_translate(p,
5897                                               action,
5898                                               &tokens[tpos],
5899                                               n_tokens - tpos,
5900                                               instr,
5901                                               data);
5902
5903         CHECK(0, EINVAL);
5904 }
5905
5906 static struct instruction_data *
5907 label_find(struct instruction_data *data, uint32_t n, const char *label)
5908 {
5909         uint32_t i;
5910
5911         for (i = 0; i < n; i++)
5912                 if (!strcmp(label, data[i].label))
5913                         return &data[i];
5914
5915         return NULL;
5916 }
5917
5918 static uint32_t
5919 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5920 {
5921         uint32_t count = 0, i;
5922
5923         if (!label[0])
5924                 return 0;
5925
5926         for (i = 0; i < n; i++)
5927                 if (!strcmp(label, data[i].jmp_label))
5928                         count++;
5929
5930         return count;
5931 }
5932
5933 static int
5934 instr_label_check(struct instruction_data *instruction_data,
5935                   uint32_t n_instructions)
5936 {
5937         uint32_t i;
5938
5939         /* Check that all instruction labels are unique. */
5940         for (i = 0; i < n_instructions; i++) {
5941                 struct instruction_data *data = &instruction_data[i];
5942                 char *label = data->label;
5943                 uint32_t j;
5944
5945                 if (!label[0])
5946                         continue;
5947
5948                 for (j = i + 1; j < n_instructions; j++)
5949                         CHECK(strcmp(label, data[j].label), EINVAL);
5950         }
5951
5952         /* Get users for each instruction label. */
5953         for (i = 0; i < n_instructions; i++) {
5954                 struct instruction_data *data = &instruction_data[i];
5955                 char *label = data->label;
5956
5957                 data->n_users = label_is_used(instruction_data,
5958                                               n_instructions,
5959                                               label);
5960         }
5961
5962         return 0;
5963 }
5964
5965 static int
5966 instr_jmp_resolve(struct instruction *instructions,
5967                   struct instruction_data *instruction_data,
5968                   uint32_t n_instructions)
5969 {
5970         uint32_t i;
5971
5972         for (i = 0; i < n_instructions; i++) {
5973                 struct instruction *instr = &instructions[i];
5974                 struct instruction_data *data = &instruction_data[i];
5975                 struct instruction_data *found;
5976
5977                 if (!instruction_is_jmp(instr))
5978                         continue;
5979
5980                 found = label_find(instruction_data,
5981                                    n_instructions,
5982                                    data->jmp_label);
5983                 CHECK(found, EINVAL);
5984
5985                 instr->jmp.ip = &instructions[found - instruction_data];
5986         }
5987
5988         return 0;
5989 }
5990
5991 static int
5992 instr_verify(struct rte_swx_pipeline *p __rte_unused,
5993              struct action *a,
5994              struct instruction *instr,
5995              struct instruction_data *data __rte_unused,
5996              uint32_t n_instructions)
5997 {
5998         if (!a) {
5999                 enum instruction_type type;
6000                 uint32_t i;
6001
6002                 /* Check that the first instruction is rx. */
6003                 CHECK(instr[0].type == INSTR_RX, EINVAL);
6004
6005                 /* Check that there is at least one tx instruction. */
6006                 for (i = 0; i < n_instructions; i++) {
6007                         type = instr[i].type;
6008
6009                         if (instruction_is_tx(type))
6010                                 break;
6011                 }
6012                 CHECK(i < n_instructions, EINVAL);
6013
6014                 /* Check that the last instruction is either tx or unconditional
6015                  * jump.
6016                  */
6017                 type = instr[n_instructions - 1].type;
6018                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6019         }
6020
6021         if (a) {
6022                 enum instruction_type type;
6023                 uint32_t i;
6024
6025                 /* Check that there is at least one return or tx instruction. */
6026                 for (i = 0; i < n_instructions; i++) {
6027                         type = instr[i].type;
6028
6029                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
6030                                 break;
6031                 }
6032                 CHECK(i < n_instructions, EINVAL);
6033         }
6034
6035         return 0;
6036 }
6037
6038 static uint32_t
6039 instr_compact(struct instruction *instructions,
6040               struct instruction_data *instruction_data,
6041               uint32_t n_instructions)
6042 {
6043         uint32_t i, pos = 0;
6044
6045         /* Eliminate the invalid instructions that have been optimized out. */
6046         for (i = 0; i < n_instructions; i++) {
6047                 struct instruction *instr = &instructions[i];
6048                 struct instruction_data *data = &instruction_data[i];
6049
6050                 if (data->invalid)
6051                         continue;
6052
6053                 if (i != pos) {
6054                         memcpy(&instructions[pos], instr, sizeof(*instr));
6055                         memcpy(&instruction_data[pos], data, sizeof(*data));
6056                 }
6057
6058                 pos++;
6059         }
6060
6061         return pos;
6062 }
6063
6064 static int
6065 instr_pattern_extract_many_search(struct instruction *instr,
6066                                   struct instruction_data *data,
6067                                   uint32_t n_instr,
6068                                   uint32_t *n_pattern_instr)
6069 {
6070         uint32_t i;
6071
6072         for (i = 0; i < n_instr; i++) {
6073                 if (data[i].invalid)
6074                         break;
6075
6076                 if (instr[i].type != INSTR_HDR_EXTRACT)
6077                         break;
6078
6079                 if (i == RTE_DIM(instr->io.hdr.header_id))
6080                         break;
6081
6082                 if (i && data[i].n_users)
6083                         break;
6084         }
6085
6086         if (i < 2)
6087                 return 0;
6088
6089         *n_pattern_instr = i;
6090         return 1;
6091 }
6092
6093 static void
6094 instr_pattern_extract_many_replace(struct instruction *instr,
6095                                    struct instruction_data *data,
6096                                    uint32_t n_instr)
6097 {
6098         uint32_t i;
6099
6100         for (i = 1; i < n_instr; i++) {
6101                 instr[0].type++;
6102                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6103                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6104                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6105
6106                 data[i].invalid = 1;
6107         }
6108 }
6109
6110 static uint32_t
6111 instr_pattern_extract_many_optimize(struct instruction *instructions,
6112                                     struct instruction_data *instruction_data,
6113                                     uint32_t n_instructions)
6114 {
6115         uint32_t i;
6116
6117         for (i = 0; i < n_instructions; ) {
6118                 struct instruction *instr = &instructions[i];
6119                 struct instruction_data *data = &instruction_data[i];
6120                 uint32_t n_instr = 0;
6121                 int detected;
6122
6123                 /* Extract many. */
6124                 detected = instr_pattern_extract_many_search(instr,
6125                                                              data,
6126                                                              n_instructions - i,
6127                                                              &n_instr);
6128                 if (detected) {
6129                         instr_pattern_extract_many_replace(instr,
6130                                                            data,
6131                                                            n_instr);
6132                         i += n_instr;
6133                         continue;
6134                 }
6135
6136                 /* No pattern starting at the current instruction. */
6137                 i++;
6138         }
6139
6140         /* Eliminate the invalid instructions that have been optimized out. */
6141         n_instructions = instr_compact(instructions,
6142                                        instruction_data,
6143                                        n_instructions);
6144
6145         return n_instructions;
6146 }
6147
6148 static int
6149 instr_pattern_emit_many_tx_search(struct instruction *instr,
6150                                   struct instruction_data *data,
6151                                   uint32_t n_instr,
6152                                   uint32_t *n_pattern_instr)
6153 {
6154         uint32_t i;
6155
6156         for (i = 0; i < n_instr; i++) {
6157                 if (data[i].invalid)
6158                         break;
6159
6160                 if (instr[i].type != INSTR_HDR_EMIT)
6161                         break;
6162
6163                 if (i == RTE_DIM(instr->io.hdr.header_id))
6164                         break;
6165
6166                 if (i && data[i].n_users)
6167                         break;
6168         }
6169
6170         if (!i)
6171                 return 0;
6172
6173         if (!instruction_is_tx(instr[i].type))
6174                 return 0;
6175
6176         if (data[i].n_users)
6177                 return 0;
6178
6179         i++;
6180
6181         *n_pattern_instr = i;
6182         return 1;
6183 }
6184
6185 static void
6186 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6187                                    struct instruction_data *data,
6188                                    uint32_t n_instr)
6189 {
6190         uint32_t i;
6191
6192         /* Any emit instruction in addition to the first one. */
6193         for (i = 1; i < n_instr - 1; i++) {
6194                 instr[0].type++;
6195                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6196                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6197                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6198
6199                 data[i].invalid = 1;
6200         }
6201
6202         /* The TX instruction is the last one in the pattern. */
6203         instr[0].type++;
6204         instr[0].io.io.offset = instr[i].io.io.offset;
6205         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6206         data[i].invalid = 1;
6207 }
6208
6209 static uint32_t
6210 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6211                                     struct instruction_data *instruction_data,
6212                                     uint32_t n_instructions)
6213 {
6214         uint32_t i;
6215
6216         for (i = 0; i < n_instructions; ) {
6217                 struct instruction *instr = &instructions[i];
6218                 struct instruction_data *data = &instruction_data[i];
6219                 uint32_t n_instr = 0;
6220                 int detected;
6221
6222                 /* Emit many + TX. */
6223                 detected = instr_pattern_emit_many_tx_search(instr,
6224                                                              data,
6225                                                              n_instructions - i,
6226                                                              &n_instr);
6227                 if (detected) {
6228                         instr_pattern_emit_many_tx_replace(instr,
6229                                                            data,
6230                                                            n_instr);
6231                         i += n_instr;
6232                         continue;
6233                 }
6234
6235                 /* No pattern starting at the current instruction. */
6236                 i++;
6237         }
6238
6239         /* Eliminate the invalid instructions that have been optimized out. */
6240         n_instructions = instr_compact(instructions,
6241                                        instruction_data,
6242                                        n_instructions);
6243
6244         return n_instructions;
6245 }
6246
6247 static uint32_t
6248 action_arg_src_mov_count(struct action *a,
6249                          uint32_t arg_id,
6250                          struct instruction *instructions,
6251                          struct instruction_data *instruction_data,
6252                          uint32_t n_instructions);
6253
6254 static int
6255 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6256                                       struct action *a,
6257                                       struct instruction *instr,
6258                                       struct instruction_data *data,
6259                                       uint32_t n_instr,
6260                                       struct instruction *instructions,
6261                                       struct instruction_data *instruction_data,
6262                                       uint32_t n_instructions,
6263                                       uint32_t *n_pattern_instr)
6264 {
6265         struct header *h;
6266         uint32_t src_field_id, i, j;
6267
6268         /* Prerequisites. */
6269         if (!a || !a->st)
6270                 return 0;
6271
6272         /* First instruction: MOV_HM. */
6273         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6274                 return 0;
6275
6276         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6277         if (!h || h->st->var_size)
6278                 return 0;
6279
6280         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6281                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6282                         break;
6283
6284         if (src_field_id == a->st->n_fields)
6285                 return 0;
6286
6287         if (instr[0].mov.dst.offset ||
6288             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6289             instr[0].mov.src.struct_id ||
6290             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6291             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6292                 return 0;
6293
6294         if ((n_instr < h->st->n_fields + 1) ||
6295              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6296                 return 0;
6297
6298         /* Subsequent instructions: MOV_HM. */
6299         for (i = 1; i < h->st->n_fields; i++)
6300                 if (data[i].invalid ||
6301                     data[i].n_users ||
6302                     (instr[i].type != INSTR_MOV_HM) ||
6303                     (instr[i].mov.dst.struct_id != h->struct_id) ||
6304                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6305                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6306                     instr[i].mov.src.struct_id ||
6307                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6308                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6309                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6310                         return 0;
6311
6312         /* Last instruction: HDR_VALIDATE. */
6313         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6314             (instr[i].valid.header_id != h->id))
6315                 return 0;
6316
6317         /* Check that none of the action args that are used as source for this
6318          * DMA transfer are not used as source in any other mov instruction.
6319          */
6320         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6321                 uint32_t n_users;
6322
6323                 n_users = action_arg_src_mov_count(a,
6324                                                    j,
6325                                                    instructions,
6326                                                    instruction_data,
6327                                                    n_instructions);
6328                 if (n_users > 1)
6329                         return 0;
6330         }
6331
6332         *n_pattern_instr = 1 + i;
6333         return 1;
6334 }
6335
6336 static void
6337 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6338                                        struct action *a,
6339                                        struct instruction *instr,
6340                                        struct instruction_data *data,
6341                                        uint32_t n_instr)
6342 {
6343         struct header *h;
6344         uint32_t src_field_id, src_offset, i;
6345
6346         /* Read from the instructions before they are modified. */
6347         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6348         if (!h)
6349                 return;
6350
6351         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6352                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6353                         break;
6354
6355         if (src_field_id == a->st->n_fields)
6356                 return;
6357
6358         src_offset = instr[0].mov.src.offset;
6359
6360         /* Modify the instructions. */
6361         instr[0].type = INSTR_DMA_HT;
6362         instr[0].dma.dst.header_id[0] = h->id;
6363         instr[0].dma.dst.struct_id[0] = h->struct_id;
6364         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6365         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6366
6367         for (i = 1; i < n_instr; i++)
6368                 data[i].invalid = 1;
6369
6370         /* Update the endianness of the action arguments to header endianness. */
6371         for (i = 0; i < h->st->n_fields; i++)
6372                 a->args_endianness[src_field_id + i] = 1;
6373 }
6374
6375 static uint32_t
6376 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6377                                         struct action *a,
6378                                         struct instruction *instructions,
6379                                         struct instruction_data *instruction_data,
6380                                         uint32_t n_instructions)
6381 {
6382         uint32_t i;
6383
6384         if (!a || !a->st)
6385                 return n_instructions;
6386
6387         for (i = 0; i < n_instructions; ) {
6388                 struct instruction *instr = &instructions[i];
6389                 struct instruction_data *data = &instruction_data[i];
6390                 uint32_t n_instr = 0;
6391                 int detected;
6392
6393                 /* Mov all + validate. */
6394                 detected = instr_pattern_mov_all_validate_search(p,
6395                                                                  a,
6396                                                                  instr,
6397                                                                  data,
6398                                                                  n_instructions - i,
6399                                                                  instructions,
6400                                                                  instruction_data,
6401                                                                  n_instructions,
6402                                                                  &n_instr);
6403                 if (detected) {
6404                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6405                         i += n_instr;
6406                         continue;
6407                 }
6408
6409                 /* No pattern starting at the current instruction. */
6410                 i++;
6411         }
6412
6413         /* Eliminate the invalid instructions that have been optimized out. */
6414         n_instructions = instr_compact(instructions,
6415                                        instruction_data,
6416                                        n_instructions);
6417
6418         return n_instructions;
6419 }
6420
6421 static int
6422 instr_pattern_dma_many_search(struct instruction *instr,
6423                               struct instruction_data *data,
6424                               uint32_t n_instr,
6425                               uint32_t *n_pattern_instr)
6426 {
6427         uint32_t i;
6428
6429         for (i = 0; i < n_instr; i++) {
6430                 if (data[i].invalid)
6431                         break;
6432
6433                 if (instr[i].type != INSTR_DMA_HT)
6434                         break;
6435
6436                 if (i == RTE_DIM(instr->dma.dst.header_id))
6437                         break;
6438
6439                 if (i && data[i].n_users)
6440                         break;
6441         }
6442
6443         if (i < 2)
6444                 return 0;
6445
6446         *n_pattern_instr = i;
6447         return 1;
6448 }
6449
6450 static void
6451 instr_pattern_dma_many_replace(struct instruction *instr,
6452                                struct instruction_data *data,
6453                                uint32_t n_instr)
6454 {
6455         uint32_t i;
6456
6457         for (i = 1; i < n_instr; i++) {
6458                 instr[0].type++;
6459                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6460                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6461                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6462                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6463
6464                 data[i].invalid = 1;
6465         }
6466 }
6467
6468 static uint32_t
6469 instr_pattern_dma_many_optimize(struct instruction *instructions,
6470                struct instruction_data *instruction_data,
6471                uint32_t n_instructions)
6472 {
6473         uint32_t i;
6474
6475         for (i = 0; i < n_instructions; ) {
6476                 struct instruction *instr = &instructions[i];
6477                 struct instruction_data *data = &instruction_data[i];
6478                 uint32_t n_instr = 0;
6479                 int detected;
6480
6481                 /* DMA many. */
6482                 detected = instr_pattern_dma_many_search(instr,
6483                                                          data,
6484                                                          n_instructions - i,
6485                                                          &n_instr);
6486                 if (detected) {
6487                         instr_pattern_dma_many_replace(instr, data, n_instr);
6488                         i += n_instr;
6489                         continue;
6490                 }
6491
6492                 /* No pattern starting at the current instruction. */
6493                 i++;
6494         }
6495
6496         /* Eliminate the invalid instructions that have been optimized out. */
6497         n_instructions = instr_compact(instructions,
6498                                        instruction_data,
6499                                        n_instructions);
6500
6501         return n_instructions;
6502 }
6503
6504 static uint32_t
6505 instr_optimize(struct rte_swx_pipeline *p,
6506                struct action *a,
6507                struct instruction *instructions,
6508                struct instruction_data *instruction_data,
6509                uint32_t n_instructions)
6510 {
6511         /* Extract many. */
6512         n_instructions = instr_pattern_extract_many_optimize(instructions,
6513                                                              instruction_data,
6514                                                              n_instructions);
6515
6516         /* Emit many + TX. */
6517         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6518                                                              instruction_data,
6519                                                              n_instructions);
6520
6521         /* Mov all + validate. */
6522         n_instructions = instr_pattern_mov_all_validate_optimize(p,
6523                                                                  a,
6524                                                                  instructions,
6525                                                                  instruction_data,
6526                                                                  n_instructions);
6527
6528         /* DMA many. */
6529         n_instructions = instr_pattern_dma_many_optimize(instructions,
6530                                                          instruction_data,
6531                                                          n_instructions);
6532
6533         return n_instructions;
6534 }
6535
6536 static int
6537 instruction_config(struct rte_swx_pipeline *p,
6538                    struct action *a,
6539                    const char **instructions,
6540                    uint32_t n_instructions)
6541 {
6542         struct instruction *instr = NULL;
6543         struct instruction_data *data = NULL;
6544         int err = 0;
6545         uint32_t i;
6546
6547         CHECK(n_instructions, EINVAL);
6548         CHECK(instructions, EINVAL);
6549         for (i = 0; i < n_instructions; i++)
6550                 CHECK_INSTRUCTION(instructions[i], EINVAL);
6551
6552         /* Memory allocation. */
6553         instr = calloc(n_instructions, sizeof(struct instruction));
6554         if (!instr) {
6555                 err = -ENOMEM;
6556                 goto error;
6557         }
6558
6559         data = calloc(n_instructions, sizeof(struct instruction_data));
6560         if (!data) {
6561                 err = -ENOMEM;
6562                 goto error;
6563         }
6564
6565         for (i = 0; i < n_instructions; i++) {
6566                 char *string = strdup(instructions[i]);
6567                 if (!string) {
6568                         err = -ENOMEM;
6569                         goto error;
6570                 }
6571
6572                 err = instr_translate(p, a, string, &instr[i], &data[i]);
6573                 if (err) {
6574                         free(string);
6575                         goto error;
6576                 }
6577
6578                 free(string);
6579         }
6580
6581         err = instr_label_check(data, n_instructions);
6582         if (err)
6583                 goto error;
6584
6585         err = instr_verify(p, a, instr, data, n_instructions);
6586         if (err)
6587                 goto error;
6588
6589         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6590
6591         err = instr_jmp_resolve(instr, data, n_instructions);
6592         if (err)
6593                 goto error;
6594
6595         if (a) {
6596                 a->instructions = instr;
6597                 a->instruction_data = data;
6598                 a->n_instructions = n_instructions;
6599         } else {
6600                 p->instructions = instr;
6601                 p->instruction_data = data;
6602                 p->n_instructions = n_instructions;
6603         }
6604
6605         return 0;
6606
6607 error:
6608         free(data);
6609         free(instr);
6610         return err;
6611 }
6612
6613 static instr_exec_t instruction_table[] = {
6614         [INSTR_RX] = instr_rx_exec,
6615         [INSTR_TX] = instr_tx_exec,
6616         [INSTR_TX_I] = instr_tx_i_exec,
6617
6618         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6619         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6620         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6621         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6622         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6623         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6624         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6625         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6626         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6627         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6628
6629         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6630         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6631         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6632         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6633         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6634         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6635         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6636         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6637         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6638
6639         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6640         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6641
6642         [INSTR_MOV] = instr_mov_exec,
6643         [INSTR_MOV_MH] = instr_mov_mh_exec,
6644         [INSTR_MOV_HM] = instr_mov_hm_exec,
6645         [INSTR_MOV_HH] = instr_mov_hh_exec,
6646         [INSTR_MOV_I] = instr_mov_i_exec,
6647
6648         [INSTR_DMA_HT] = instr_dma_ht_exec,
6649         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6650         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6651         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6652         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6653         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6654         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6655         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6656
6657         [INSTR_ALU_ADD] = instr_alu_add_exec,
6658         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6659         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6660         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6661         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6662         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6663
6664         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6665         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6666         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6667         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6668         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6669         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6670
6671         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6672         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6673         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6674         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6675
6676         [INSTR_ALU_AND] = instr_alu_and_exec,
6677         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6678         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6679         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6680         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6681
6682         [INSTR_ALU_OR] = instr_alu_or_exec,
6683         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6684         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6685         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6686         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6687
6688         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6689         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6690         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6691         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6692         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6693
6694         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6695         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6696         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6697         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6698         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6699         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6700
6701         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6702         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6703         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6704         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6705         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6706         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6707
6708         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6709         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6710         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6711
6712         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6713         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6714         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6715         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6716         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6717         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6718
6719         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6720         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6721         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6722         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6723         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6724         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6725         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6726         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6727         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6728
6729         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6730         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6731         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6732         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6733         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6734         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6735         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6736         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6737         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6738
6739         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6740         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6741         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6742
6743         [INSTR_METER_HHM] = instr_meter_hhm_exec,
6744         [INSTR_METER_HHI] = instr_meter_hhi_exec,
6745         [INSTR_METER_HMM] = instr_meter_hmm_exec,
6746         [INSTR_METER_HMI] = instr_meter_hmi_exec,
6747         [INSTR_METER_MHM] = instr_meter_mhm_exec,
6748         [INSTR_METER_MHI] = instr_meter_mhi_exec,
6749         [INSTR_METER_MMM] = instr_meter_mmm_exec,
6750         [INSTR_METER_MMI] = instr_meter_mmi_exec,
6751         [INSTR_METER_IHM] = instr_meter_ihm_exec,
6752         [INSTR_METER_IHI] = instr_meter_ihi_exec,
6753         [INSTR_METER_IMM] = instr_meter_imm_exec,
6754         [INSTR_METER_IMI] = instr_meter_imi_exec,
6755
6756         [INSTR_TABLE] = instr_table_exec,
6757         [INSTR_TABLE_AF] = instr_table_af_exec,
6758         [INSTR_SELECTOR] = instr_selector_exec,
6759         [INSTR_LEARNER] = instr_learner_exec,
6760         [INSTR_LEARNER_AF] = instr_learner_af_exec,
6761         [INSTR_LEARNER_LEARN] = instr_learn_exec,
6762         [INSTR_LEARNER_FORGET] = instr_forget_exec,
6763         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6764         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6765
6766         [INSTR_JMP] = instr_jmp_exec,
6767         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6768         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6769         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6770         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6771         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6772         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6773
6774         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6775         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6776         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6777         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6778         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6779
6780         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6781         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6782         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6783         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6784         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6785
6786         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6787         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6788         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6789         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6790         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6791         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6792
6793         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6794         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6795         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6796         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6797         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6798         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6799
6800         [INSTR_RETURN] = instr_return_exec,
6801 };
6802
6803 static int
6804 instruction_table_build(struct rte_swx_pipeline *p)
6805 {
6806         p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6807                                       sizeof(struct instr_exec_t *));
6808         if (!p->instruction_table)
6809                 return -EINVAL;
6810
6811         memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6812
6813         return 0;
6814 }
6815
6816 static void
6817 instruction_table_build_free(struct rte_swx_pipeline *p)
6818 {
6819         if (!p->instruction_table)
6820                 return;
6821
6822         free(p->instruction_table);
6823         p->instruction_table = NULL;
6824 }
6825
6826 static void
6827 instruction_table_free(struct rte_swx_pipeline *p)
6828 {
6829         instruction_table_build_free(p);
6830 }
6831
6832 static inline void
6833 instr_exec(struct rte_swx_pipeline *p)
6834 {
6835         struct thread *t = &p->threads[p->thread_id];
6836         struct instruction *ip = t->ip;
6837         instr_exec_t instr = p->instruction_table[ip->type];
6838
6839         instr(p);
6840 }
6841
6842 /*
6843  * Action.
6844  */
6845 static struct action *
6846 action_find(struct rte_swx_pipeline *p, const char *name)
6847 {
6848         struct action *elem;
6849
6850         if (!name)
6851                 return NULL;
6852
6853         TAILQ_FOREACH(elem, &p->actions, node)
6854                 if (strcmp(elem->name, name) == 0)
6855                         return elem;
6856
6857         return NULL;
6858 }
6859
6860 static struct action *
6861 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6862 {
6863         struct action *action = NULL;
6864
6865         TAILQ_FOREACH(action, &p->actions, node)
6866                 if (action->id == id)
6867                         return action;
6868
6869         return NULL;
6870 }
6871
6872 static struct field *
6873 action_field_find(struct action *a, const char *name)
6874 {
6875         return a->st ? struct_type_field_find(a->st, name) : NULL;
6876 }
6877
6878 static struct field *
6879 action_field_parse(struct action *action, const char *name)
6880 {
6881         if (name[0] != 't' || name[1] != '.')
6882                 return NULL;
6883
6884         return action_field_find(action, &name[2]);
6885 }
6886
6887 static int
6888 action_has_nbo_args(struct action *a)
6889 {
6890         uint32_t i;
6891
6892         /* Return if the action does not have any args. */
6893         if (!a->st)
6894                 return 0; /* FALSE */
6895
6896         for (i = 0; i < a->st->n_fields; i++)
6897                 if (a->args_endianness[i])
6898                         return 1; /* TRUE */
6899
6900         return 0; /* FALSE */
6901 }
6902
6903 static int
6904 action_does_learning(struct action *a)
6905 {
6906         uint32_t i;
6907
6908         for (i = 0; i < a->n_instructions; i++)
6909                 switch (a->instructions[i].type) {
6910                 case INSTR_LEARNER_LEARN:
6911                         return 1; /* TRUE */
6912
6913                 case INSTR_LEARNER_FORGET:
6914                         return 1; /* TRUE */
6915
6916                 default:
6917                         continue;
6918                 }
6919
6920         return 0; /* FALSE */
6921 }
6922
6923 int
6924 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6925                                const char *name,
6926                                const char *args_struct_type_name,
6927                                const char **instructions,
6928                                uint32_t n_instructions)
6929 {
6930         struct struct_type *args_struct_type = NULL;
6931         struct action *a;
6932         int err;
6933
6934         CHECK(p, EINVAL);
6935
6936         CHECK_NAME(name, EINVAL);
6937         CHECK(!action_find(p, name), EEXIST);
6938
6939         if (args_struct_type_name) {
6940                 CHECK_NAME(args_struct_type_name, EINVAL);
6941                 args_struct_type = struct_type_find(p, args_struct_type_name);
6942                 CHECK(args_struct_type, EINVAL);
6943                 CHECK(!args_struct_type->var_size, EINVAL);
6944         }
6945
6946         /* Node allocation. */
6947         a = calloc(1, sizeof(struct action));
6948         CHECK(a, ENOMEM);
6949         if (args_struct_type) {
6950                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6951                 if (!a->args_endianness) {
6952                         free(a);
6953                         CHECK(0, ENOMEM);
6954                 }
6955         }
6956
6957         /* Node initialization. */
6958         strcpy(a->name, name);
6959         a->st = args_struct_type;
6960         a->id = p->n_actions;
6961
6962         /* Instruction translation. */
6963         err = instruction_config(p, a, instructions, n_instructions);
6964         if (err) {
6965                 free(a->args_endianness);
6966                 free(a);
6967                 return err;
6968         }
6969
6970         /* Node add to tailq. */
6971         TAILQ_INSERT_TAIL(&p->actions, a, node);
6972         p->n_actions++;
6973
6974         return 0;
6975 }
6976
6977 static int
6978 action_build(struct rte_swx_pipeline *p)
6979 {
6980         struct action *action;
6981
6982         /* p->action_instructions. */
6983         p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
6984         CHECK(p->action_instructions, ENOMEM);
6985
6986         TAILQ_FOREACH(action, &p->actions, node)
6987                 p->action_instructions[action->id] = action->instructions;
6988
6989         /* p->action_funcs. */
6990         p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
6991         CHECK(p->action_funcs, ENOMEM);
6992
6993         return 0;
6994 }
6995
6996 static void
6997 action_build_free(struct rte_swx_pipeline *p)
6998 {
6999         free(p->action_funcs);
7000         p->action_funcs = NULL;
7001
7002         free(p->action_instructions);
7003         p->action_instructions = NULL;
7004 }
7005
7006 static void
7007 action_free(struct rte_swx_pipeline *p)
7008 {
7009         action_build_free(p);
7010
7011         for ( ; ; ) {
7012                 struct action *action;
7013
7014                 action = TAILQ_FIRST(&p->actions);
7015                 if (!action)
7016                         break;
7017
7018                 TAILQ_REMOVE(&p->actions, action, node);
7019                 free(action->instruction_data);
7020                 free(action->instructions);
7021                 free(action);
7022         }
7023 }
7024
7025 static uint32_t
7026 action_arg_src_mov_count(struct action *a,
7027                          uint32_t arg_id,
7028                          struct instruction *instructions,
7029                          struct instruction_data *instruction_data,
7030                          uint32_t n_instructions)
7031 {
7032         uint32_t offset, n_users = 0, i;
7033
7034         if (!a->st ||
7035             (arg_id >= a->st->n_fields) ||
7036             !instructions ||
7037             !instruction_data ||
7038             !n_instructions)
7039                 return 0;
7040
7041         offset = a->st->fields[arg_id].offset / 8;
7042
7043         for (i = 0; i < n_instructions; i++) {
7044                 struct instruction *instr = &instructions[i];
7045                 struct instruction_data *data = &instruction_data[i];
7046
7047                 if (data->invalid ||
7048                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7049                     instr->mov.src.struct_id ||
7050                     (instr->mov.src.offset != offset))
7051                         continue;
7052
7053                 n_users++;
7054         }
7055
7056         return n_users;
7057 }
7058
7059 /*
7060  * Table.
7061  */
7062 static struct table_type *
7063 table_type_find(struct rte_swx_pipeline *p, const char *name)
7064 {
7065         struct table_type *elem;
7066
7067         TAILQ_FOREACH(elem, &p->table_types, node)
7068                 if (strcmp(elem->name, name) == 0)
7069                         return elem;
7070
7071         return NULL;
7072 }
7073
7074 static struct table_type *
7075 table_type_resolve(struct rte_swx_pipeline *p,
7076                    const char *recommended_type_name,
7077                    enum rte_swx_table_match_type match_type)
7078 {
7079         struct table_type *elem;
7080
7081         /* Only consider the recommended type if the match type is correct. */
7082         if (recommended_type_name)
7083                 TAILQ_FOREACH(elem, &p->table_types, node)
7084                         if (!strcmp(elem->name, recommended_type_name) &&
7085                             (elem->match_type == match_type))
7086                                 return elem;
7087
7088         /* Ignore the recommended type and get the first element with this match
7089          * type.
7090          */
7091         TAILQ_FOREACH(elem, &p->table_types, node)
7092                 if (elem->match_type == match_type)
7093                         return elem;
7094
7095         return NULL;
7096 }
7097
7098 static struct table *
7099 table_find(struct rte_swx_pipeline *p, const char *name)
7100 {
7101         struct table *elem;
7102
7103         TAILQ_FOREACH(elem, &p->tables, node)
7104                 if (strcmp(elem->name, name) == 0)
7105                         return elem;
7106
7107         return NULL;
7108 }
7109
7110 static struct table *
7111 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7112 {
7113         struct table *table = NULL;
7114
7115         TAILQ_FOREACH(table, &p->tables, node)
7116                 if (table->id == id)
7117                         return table;
7118
7119         return NULL;
7120 }
7121
7122 int
7123 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7124                                      const char *name,
7125                                      enum rte_swx_table_match_type match_type,
7126                                      struct rte_swx_table_ops *ops)
7127 {
7128         struct table_type *elem;
7129
7130         CHECK(p, EINVAL);
7131
7132         CHECK_NAME(name, EINVAL);
7133         CHECK(!table_type_find(p, name), EEXIST);
7134
7135         CHECK(ops, EINVAL);
7136         CHECK(ops->create, EINVAL);
7137         CHECK(ops->lkp, EINVAL);
7138         CHECK(ops->free, EINVAL);
7139
7140         /* Node allocation. */
7141         elem = calloc(1, sizeof(struct table_type));
7142         CHECK(elem, ENOMEM);
7143
7144         /* Node initialization. */
7145         strcpy(elem->name, name);
7146         elem->match_type = match_type;
7147         memcpy(&elem->ops, ops, sizeof(*ops));
7148
7149         /* Node add to tailq. */
7150         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7151
7152         return 0;
7153 }
7154
7155 static int
7156 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7157                          uint32_t n_fields,
7158                          enum rte_swx_table_match_type *match_type)
7159 {
7160         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7161
7162         for (i = 0; i < n_fields; i++) {
7163                 struct rte_swx_match_field_params  *f = &fields[i];
7164
7165                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7166                         n_fields_em++;
7167
7168                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7169                         n_fields_lpm++;
7170         }
7171
7172         if ((n_fields_lpm > 1) ||
7173             (n_fields_lpm && (n_fields_em != n_fields - 1)))
7174                 return -EINVAL;
7175
7176         *match_type = (n_fields_em == n_fields) ?
7177                        RTE_SWX_TABLE_MATCH_EXACT :
7178                        RTE_SWX_TABLE_MATCH_WILDCARD;
7179
7180         return 0;
7181 }
7182
7183 static int
7184 table_match_fields_check(struct rte_swx_pipeline *p,
7185                          struct rte_swx_pipeline_table_params *params,
7186                          struct header **header)
7187 {
7188         struct header *h0 = NULL;
7189         struct field *hf, *mf;
7190         uint32_t *offset = NULL, i;
7191         int status = 0;
7192
7193         /* Return if no match fields. */
7194         if (!params->n_fields) {
7195                 if (params->fields) {
7196                         status = -EINVAL;
7197                         goto end;
7198                 }
7199
7200                 if (header)
7201                         *header = NULL;
7202
7203                 return 0;
7204         }
7205
7206         /* Memory allocation. */
7207         offset = calloc(params->n_fields, sizeof(uint32_t));
7208         if (!offset) {
7209                 status = -ENOMEM;
7210                 goto end;
7211         }
7212
7213         /* Check that all the match fields belong to either the same header or
7214          * to the meta-data.
7215          */
7216         hf = header_field_parse(p, params->fields[0].name, &h0);
7217         mf = metadata_field_parse(p, params->fields[0].name);
7218         if ((!hf && !mf) || (hf && hf->var_size)) {
7219                 status = -EINVAL;
7220                 goto end;
7221         }
7222
7223         offset[0] = h0 ? hf->offset : mf->offset;
7224
7225         for (i = 1; i < params->n_fields; i++)
7226                 if (h0) {
7227                         struct header *h;
7228
7229                         hf = header_field_parse(p, params->fields[i].name, &h);
7230                         if (!hf || (h->id != h0->id) || hf->var_size) {
7231                                 status = -EINVAL;
7232                                 goto end;
7233                         }
7234
7235                         offset[i] = hf->offset;
7236                 } else {
7237                         mf = metadata_field_parse(p, params->fields[i].name);
7238                         if (!mf) {
7239                                 status = -EINVAL;
7240                                 goto end;
7241                         }
7242
7243                         offset[i] = mf->offset;
7244                 }
7245
7246         /* Check that there are no duplicated match fields. */
7247         for (i = 0; i < params->n_fields; i++) {
7248                 uint32_t j;
7249
7250                 for (j = 0; j < i; j++)
7251                         if (offset[j] == offset[i]) {
7252                                 status = -EINVAL;
7253                                 goto end;
7254                         }
7255         }
7256
7257         /* Return. */
7258         if (header)
7259                 *header = h0;
7260
7261 end:
7262         free(offset);
7263         return status;
7264 }
7265
7266 int
7267 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7268                               const char *name,
7269                               struct rte_swx_pipeline_table_params *params,
7270                               const char *recommended_table_type_name,
7271                               const char *args,
7272                               uint32_t size)
7273 {
7274         struct table_type *type;
7275         struct table *t;
7276         struct action *default_action;
7277         struct header *header = NULL;
7278         uint32_t action_data_size_max = 0, i;
7279         int status = 0;
7280
7281         CHECK(p, EINVAL);
7282
7283         CHECK_NAME(name, EINVAL);
7284         CHECK(!table_find(p, name), EEXIST);
7285         CHECK(!selector_find(p, name), EEXIST);
7286         CHECK(!learner_find(p, name), EEXIST);
7287
7288         CHECK(params, EINVAL);
7289
7290         /* Match checks. */
7291         status = table_match_fields_check(p, params, &header);
7292         if (status)
7293                 return status;
7294
7295         /* Action checks. */
7296         CHECK(params->n_actions, EINVAL);
7297         CHECK(params->action_names, EINVAL);
7298         for (i = 0; i < params->n_actions; i++) {
7299                 const char *action_name = params->action_names[i];
7300                 struct action *a;
7301                 uint32_t action_data_size;
7302
7303                 CHECK_NAME(action_name, EINVAL);
7304
7305                 a = action_find(p, action_name);
7306                 CHECK(a, EINVAL);
7307                 CHECK(!action_does_learning(a), EINVAL);
7308
7309                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7310                 if (action_data_size > action_data_size_max)
7311                         action_data_size_max = action_data_size;
7312         }
7313
7314         CHECK_NAME(params->default_action_name, EINVAL);
7315         for (i = 0; i < p->n_actions; i++)
7316                 if (!strcmp(params->action_names[i],
7317                             params->default_action_name))
7318                         break;
7319         CHECK(i < params->n_actions, EINVAL);
7320         default_action = action_find(p, params->default_action_name);
7321         CHECK((default_action->st && params->default_action_data) ||
7322               !params->default_action_data, EINVAL);
7323
7324         /* Table type checks. */
7325         if (recommended_table_type_name)
7326                 CHECK_NAME(recommended_table_type_name, EINVAL);
7327
7328         if (params->n_fields) {
7329                 enum rte_swx_table_match_type match_type;
7330
7331                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7332                 if (status)
7333                         return status;
7334
7335                 type = table_type_resolve(p, recommended_table_type_name, match_type);
7336                 CHECK(type, EINVAL);
7337         } else {
7338                 type = NULL;
7339         }
7340
7341         /* Memory allocation. */
7342         t = calloc(1, sizeof(struct table));
7343         CHECK(t, ENOMEM);
7344
7345         t->fields = calloc(params->n_fields, sizeof(struct match_field));
7346         if (!t->fields) {
7347                 free(t);
7348                 CHECK(0, ENOMEM);
7349         }
7350
7351         t->actions = calloc(params->n_actions, sizeof(struct action *));
7352         if (!t->actions) {
7353                 free(t->fields);
7354                 free(t);
7355                 CHECK(0, ENOMEM);
7356         }
7357
7358         if (action_data_size_max) {
7359                 t->default_action_data = calloc(1, action_data_size_max);
7360                 if (!t->default_action_data) {
7361                         free(t->actions);
7362                         free(t->fields);
7363                         free(t);
7364                         CHECK(0, ENOMEM);
7365                 }
7366         }
7367
7368         /* Node initialization. */
7369         strcpy(t->name, name);
7370         if (args && args[0])
7371                 strcpy(t->args, args);
7372         t->type = type;
7373
7374         for (i = 0; i < params->n_fields; i++) {
7375                 struct rte_swx_match_field_params *field = &params->fields[i];
7376                 struct match_field *f = &t->fields[i];
7377
7378                 f->match_type = field->match_type;
7379                 f->field = header ?
7380                         header_field_parse(p, field->name, NULL) :
7381                         metadata_field_parse(p, field->name);
7382         }
7383         t->n_fields = params->n_fields;
7384         t->header = header;
7385
7386         for (i = 0; i < params->n_actions; i++)
7387                 t->actions[i] = action_find(p, params->action_names[i]);
7388         t->default_action = default_action;
7389         if (default_action->st)
7390                 memcpy(t->default_action_data,
7391                        params->default_action_data,
7392                        default_action->st->n_bits / 8);
7393         t->n_actions = params->n_actions;
7394         t->default_action_is_const = params->default_action_is_const;
7395         t->action_data_size_max = action_data_size_max;
7396
7397         t->size = size;
7398         t->id = p->n_tables;
7399
7400         /* Node add to tailq. */
7401         TAILQ_INSERT_TAIL(&p->tables, t, node);
7402         p->n_tables++;
7403
7404         return 0;
7405 }
7406
7407 static struct rte_swx_table_params *
7408 table_params_get(struct table *table)
7409 {
7410         struct rte_swx_table_params *params;
7411         struct field *first, *last;
7412         uint8_t *key_mask;
7413         uint32_t key_size, key_offset, action_data_size, i;
7414
7415         /* Memory allocation. */
7416         params = calloc(1, sizeof(struct rte_swx_table_params));
7417         if (!params)
7418                 return NULL;
7419
7420         /* Find first (smallest offset) and last (biggest offset) match fields. */
7421         first = table->fields[0].field;
7422         last = table->fields[0].field;
7423
7424         for (i = 0; i < table->n_fields; i++) {
7425                 struct field *f = table->fields[i].field;
7426
7427                 if (f->offset < first->offset)
7428                         first = f;
7429
7430                 if (f->offset > last->offset)
7431                         last = f;
7432         }
7433
7434         /* Key offset and size. */
7435         key_offset = first->offset / 8;
7436         key_size = (last->offset + last->n_bits - first->offset) / 8;
7437
7438         /* Memory allocation. */
7439         key_mask = calloc(1, key_size);
7440         if (!key_mask) {
7441                 free(params);
7442                 return NULL;
7443         }
7444
7445         /* Key mask. */
7446         for (i = 0; i < table->n_fields; i++) {
7447                 struct field *f = table->fields[i].field;
7448                 uint32_t start = (f->offset - first->offset) / 8;
7449                 size_t size = f->n_bits / 8;
7450
7451                 memset(&key_mask[start], 0xFF, size);
7452         }
7453
7454         /* Action data size. */
7455         action_data_size = 0;
7456         for (i = 0; i < table->n_actions; i++) {
7457                 struct action *action = table->actions[i];
7458                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7459
7460                 if (ads > action_data_size)
7461                         action_data_size = ads;
7462         }
7463
7464         /* Fill in. */
7465         params->match_type = table->type->match_type;
7466         params->key_size = key_size;
7467         params->key_offset = key_offset;
7468         params->key_mask0 = key_mask;
7469         params->action_data_size = action_data_size;
7470         params->n_keys_max = table->size;
7471
7472         return params;
7473 }
7474
7475 static void
7476 table_params_free(struct rte_swx_table_params *params)
7477 {
7478         if (!params)
7479                 return;
7480
7481         free(params->key_mask0);
7482         free(params);
7483 }
7484
7485 static int
7486 table_stub_lkp(void *table __rte_unused,
7487                void *mailbox __rte_unused,
7488                uint8_t **key __rte_unused,
7489                uint64_t *action_id __rte_unused,
7490                uint8_t **action_data __rte_unused,
7491                int *hit)
7492 {
7493         *hit = 0;
7494         return 1; /* DONE. */
7495 }
7496
7497 static int
7498 table_build(struct rte_swx_pipeline *p)
7499 {
7500         uint32_t i;
7501
7502         /* Per pipeline: table statistics. */
7503         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7504         CHECK(p->table_stats, ENOMEM);
7505
7506         for (i = 0; i < p->n_tables; i++) {
7507                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7508                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7509         }
7510
7511         /* Per thread: table runt-time. */
7512         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7513                 struct thread *t = &p->threads[i];
7514                 struct table *table;
7515
7516                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7517                 CHECK(t->tables, ENOMEM);
7518
7519                 TAILQ_FOREACH(table, &p->tables, node) {
7520                         struct table_runtime *r = &t->tables[table->id];
7521
7522                         if (table->type) {
7523                                 uint64_t size;
7524
7525                                 size = table->type->ops.mailbox_size_get();
7526
7527                                 /* r->func. */
7528                                 r->func = table->type->ops.lkp;
7529
7530                                 /* r->mailbox. */
7531                                 if (size) {
7532                                         r->mailbox = calloc(1, size);
7533                                         CHECK(r->mailbox, ENOMEM);
7534                                 }
7535
7536                                 /* r->key. */
7537                                 r->key = table->header ?
7538                                         &t->structs[table->header->struct_id] :
7539                                         &t->structs[p->metadata_struct_id];
7540                         } else {
7541                                 r->func = table_stub_lkp;
7542                         }
7543                 }
7544         }
7545
7546         return 0;
7547 }
7548
7549 static void
7550 table_build_free(struct rte_swx_pipeline *p)
7551 {
7552         uint32_t i;
7553
7554         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7555                 struct thread *t = &p->threads[i];
7556                 uint32_t j;
7557
7558                 if (!t->tables)
7559                         continue;
7560
7561                 for (j = 0; j < p->n_tables; j++) {
7562                         struct table_runtime *r = &t->tables[j];
7563
7564                         free(r->mailbox);
7565                 }
7566
7567                 free(t->tables);
7568                 t->tables = NULL;
7569         }
7570
7571         if (p->table_stats) {
7572                 for (i = 0; i < p->n_tables; i++)
7573                         free(p->table_stats[i].n_pkts_action);
7574
7575                 free(p->table_stats);
7576         }
7577 }
7578
7579 static void
7580 table_free(struct rte_swx_pipeline *p)
7581 {
7582         table_build_free(p);
7583
7584         /* Tables. */
7585         for ( ; ; ) {
7586                 struct table *elem;
7587
7588                 elem = TAILQ_FIRST(&p->tables);
7589                 if (!elem)
7590                         break;
7591
7592                 TAILQ_REMOVE(&p->tables, elem, node);
7593                 free(elem->fields);
7594                 free(elem->actions);
7595                 free(elem->default_action_data);
7596                 free(elem);
7597         }
7598
7599         /* Table types. */
7600         for ( ; ; ) {
7601                 struct table_type *elem;
7602
7603                 elem = TAILQ_FIRST(&p->table_types);
7604                 if (!elem)
7605                         break;
7606
7607                 TAILQ_REMOVE(&p->table_types, elem, node);
7608                 free(elem);
7609         }
7610 }
7611
7612 /*
7613  * Selector.
7614  */
7615 static struct selector *
7616 selector_find(struct rte_swx_pipeline *p, const char *name)
7617 {
7618         struct selector *s;
7619
7620         TAILQ_FOREACH(s, &p->selectors, node)
7621                 if (strcmp(s->name, name) == 0)
7622                         return s;
7623
7624         return NULL;
7625 }
7626
7627 static struct selector *
7628 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7629 {
7630         struct selector *s = NULL;
7631
7632         TAILQ_FOREACH(s, &p->selectors, node)
7633                 if (s->id == id)
7634                         return s;
7635
7636         return NULL;
7637 }
7638
7639 static int
7640 selector_fields_check(struct rte_swx_pipeline *p,
7641                       struct rte_swx_pipeline_selector_params *params,
7642                       struct header **header)
7643 {
7644         struct header *h0 = NULL;
7645         struct field *hf, *mf;
7646         uint32_t i;
7647
7648         /* Return if no selector fields. */
7649         if (!params->n_selector_fields || !params->selector_field_names)
7650                 return -EINVAL;
7651
7652         /* Check that all the selector fields either belong to the same header
7653          * or are all meta-data fields.
7654          */
7655         hf = header_field_parse(p, params->selector_field_names[0], &h0);
7656         mf = metadata_field_parse(p, params->selector_field_names[0]);
7657         if (!hf && !mf)
7658                 return -EINVAL;
7659
7660         for (i = 1; i < params->n_selector_fields; i++)
7661                 if (h0) {
7662                         struct header *h;
7663
7664                         hf = header_field_parse(p, params->selector_field_names[i], &h);
7665                         if (!hf || (h->id != h0->id))
7666                                 return -EINVAL;
7667                 } else {
7668                         mf = metadata_field_parse(p, params->selector_field_names[i]);
7669                         if (!mf)
7670                                 return -EINVAL;
7671                 }
7672
7673         /* Check that there are no duplicated match fields. */
7674         for (i = 0; i < params->n_selector_fields; i++) {
7675                 const char *field_name = params->selector_field_names[i];
7676                 uint32_t j;
7677
7678                 for (j = i + 1; j < params->n_selector_fields; j++)
7679                         if (!strcmp(params->selector_field_names[j], field_name))
7680                                 return -EINVAL;
7681         }
7682
7683         /* Return. */
7684         if (header)
7685                 *header = h0;
7686
7687         return 0;
7688 }
7689
7690 int
7691 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7692                                  const char *name,
7693                                  struct rte_swx_pipeline_selector_params *params)
7694 {
7695         struct selector *s;
7696         struct header *selector_header = NULL;
7697         struct field *group_id_field, *member_id_field;
7698         uint32_t i;
7699         int status = 0;
7700
7701         CHECK(p, EINVAL);
7702
7703         CHECK_NAME(name, EINVAL);
7704         CHECK(!table_find(p, name), EEXIST);
7705         CHECK(!selector_find(p, name), EEXIST);
7706         CHECK(!learner_find(p, name), EEXIST);
7707
7708         CHECK(params, EINVAL);
7709
7710         CHECK_NAME(params->group_id_field_name, EINVAL);
7711         group_id_field = metadata_field_parse(p, params->group_id_field_name);
7712         CHECK(group_id_field, EINVAL);
7713
7714         for (i = 0; i < params->n_selector_fields; i++) {
7715                 const char *field_name = params->selector_field_names[i];
7716
7717                 CHECK_NAME(field_name, EINVAL);
7718         }
7719         status = selector_fields_check(p, params, &selector_header);
7720         if (status)
7721                 return status;
7722
7723         CHECK_NAME(params->member_id_field_name, EINVAL);
7724         member_id_field = metadata_field_parse(p, params->member_id_field_name);
7725         CHECK(member_id_field, EINVAL);
7726
7727         CHECK(params->n_groups_max, EINVAL);
7728
7729         CHECK(params->n_members_per_group_max, EINVAL);
7730
7731         /* Memory allocation. */
7732         s = calloc(1, sizeof(struct selector));
7733         if (!s) {
7734                 status = -ENOMEM;
7735                 goto error;
7736         }
7737
7738         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7739         if (!s->selector_fields) {
7740                 status = -ENOMEM;
7741                 goto error;
7742         }
7743
7744         /* Node initialization. */
7745         strcpy(s->name, name);
7746
7747         s->group_id_field = group_id_field;
7748
7749         for (i = 0; i < params->n_selector_fields; i++) {
7750                 const char *field_name = params->selector_field_names[i];
7751
7752                 s->selector_fields[i] = selector_header ?
7753                         header_field_parse(p, field_name, NULL) :
7754                         metadata_field_parse(p, field_name);
7755         }
7756
7757         s->n_selector_fields = params->n_selector_fields;
7758
7759         s->selector_header = selector_header;
7760
7761         s->member_id_field = member_id_field;
7762
7763         s->n_groups_max = params->n_groups_max;
7764
7765         s->n_members_per_group_max = params->n_members_per_group_max;
7766
7767         s->id = p->n_selectors;
7768
7769         /* Node add to tailq. */
7770         TAILQ_INSERT_TAIL(&p->selectors, s, node);
7771         p->n_selectors++;
7772
7773         return 0;
7774
7775 error:
7776         if (!s)
7777                 return status;
7778
7779         free(s->selector_fields);
7780
7781         free(s);
7782
7783         return status;
7784 }
7785
7786 static void
7787 selector_params_free(struct rte_swx_table_selector_params *params)
7788 {
7789         if (!params)
7790                 return;
7791
7792         free(params->selector_mask);
7793
7794         free(params);
7795 }
7796
7797 static struct rte_swx_table_selector_params *
7798 selector_table_params_get(struct selector *s)
7799 {
7800         struct rte_swx_table_selector_params *params = NULL;
7801         struct field *first, *last;
7802         uint32_t i;
7803
7804         /* Memory allocation. */
7805         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7806         if (!params)
7807                 goto error;
7808
7809         /* Group ID. */
7810         params->group_id_offset = s->group_id_field->offset / 8;
7811
7812         /* Find first (smallest offset) and last (biggest offset) selector fields. */
7813         first = s->selector_fields[0];
7814         last = s->selector_fields[0];
7815
7816         for (i = 0; i < s->n_selector_fields; i++) {
7817                 struct field *f = s->selector_fields[i];
7818
7819                 if (f->offset < first->offset)
7820                         first = f;
7821
7822                 if (f->offset > last->offset)
7823                         last = f;
7824         }
7825
7826         /* Selector offset and size. */
7827         params->selector_offset = first->offset / 8;
7828         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7829
7830         /* Memory allocation. */
7831         params->selector_mask = calloc(1, params->selector_size);
7832         if (!params->selector_mask)
7833                 goto error;
7834
7835         /* Selector mask. */
7836         for (i = 0; i < s->n_selector_fields; i++) {
7837                 struct field *f = s->selector_fields[i];
7838                 uint32_t start = (f->offset - first->offset) / 8;
7839                 size_t size = f->n_bits / 8;
7840
7841                 memset(&params->selector_mask[start], 0xFF, size);
7842         }
7843
7844         /* Member ID. */
7845         params->member_id_offset = s->member_id_field->offset / 8;
7846
7847         /* Maximum number of groups. */
7848         params->n_groups_max = s->n_groups_max;
7849
7850         /* Maximum number of members per group. */
7851         params->n_members_per_group_max = s->n_members_per_group_max;
7852
7853         return params;
7854
7855 error:
7856         selector_params_free(params);
7857         return NULL;
7858 }
7859
7860 static void
7861 selector_build_free(struct rte_swx_pipeline *p)
7862 {
7863         uint32_t i;
7864
7865         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7866                 struct thread *t = &p->threads[i];
7867                 uint32_t j;
7868
7869                 if (!t->selectors)
7870                         continue;
7871
7872                 for (j = 0; j < p->n_selectors; j++) {
7873                         struct selector_runtime *r = &t->selectors[j];
7874
7875                         free(r->mailbox);
7876                 }
7877
7878                 free(t->selectors);
7879                 t->selectors = NULL;
7880         }
7881
7882         free(p->selector_stats);
7883         p->selector_stats = NULL;
7884 }
7885
7886 static int
7887 selector_build(struct rte_swx_pipeline *p)
7888 {
7889         uint32_t i;
7890         int status = 0;
7891
7892         /* Per pipeline: selector statistics. */
7893         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7894         if (!p->selector_stats) {
7895                 status = -ENOMEM;
7896                 goto error;
7897         }
7898
7899         /* Per thread: selector run-time. */
7900         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7901                 struct thread *t = &p->threads[i];
7902                 struct selector *s;
7903
7904                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7905                 if (!t->selectors) {
7906                         status = -ENOMEM;
7907                         goto error;
7908                 }
7909
7910                 TAILQ_FOREACH(s, &p->selectors, node) {
7911                         struct selector_runtime *r = &t->selectors[s->id];
7912                         uint64_t size;
7913
7914                         /* r->mailbox. */
7915                         size = rte_swx_table_selector_mailbox_size_get();
7916                         if (size) {
7917                                 r->mailbox = calloc(1, size);
7918                                 if (!r->mailbox) {
7919                                         status = -ENOMEM;
7920                                         goto error;
7921                                 }
7922                         }
7923
7924                         /* r->group_id_buffer. */
7925                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
7926
7927                         /* r->selector_buffer. */
7928                         r->selector_buffer = s->selector_header ?
7929                                 &t->structs[s->selector_header->struct_id] :
7930                                 &t->structs[p->metadata_struct_id];
7931
7932                         /* r->member_id_buffer. */
7933                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
7934                 }
7935         }
7936
7937         return 0;
7938
7939 error:
7940         selector_build_free(p);
7941         return status;
7942 }
7943
7944 static void
7945 selector_free(struct rte_swx_pipeline *p)
7946 {
7947         selector_build_free(p);
7948
7949         /* Selector tables. */
7950         for ( ; ; ) {
7951                 struct selector *elem;
7952
7953                 elem = TAILQ_FIRST(&p->selectors);
7954                 if (!elem)
7955                         break;
7956
7957                 TAILQ_REMOVE(&p->selectors, elem, node);
7958                 free(elem->selector_fields);
7959                 free(elem);
7960         }
7961 }
7962
7963 /*
7964  * Learner table.
7965  */
7966 static struct learner *
7967 learner_find(struct rte_swx_pipeline *p, const char *name)
7968 {
7969         struct learner *l;
7970
7971         TAILQ_FOREACH(l, &p->learners, node)
7972                 if (!strcmp(l->name, name))
7973                         return l;
7974
7975         return NULL;
7976 }
7977
7978 static struct learner *
7979 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7980 {
7981         struct learner *l = NULL;
7982
7983         TAILQ_FOREACH(l, &p->learners, node)
7984                 if (l->id == id)
7985                         return l;
7986
7987         return NULL;
7988 }
7989
7990 static int
7991 learner_match_fields_check(struct rte_swx_pipeline *p,
7992                            struct rte_swx_pipeline_learner_params *params,
7993                            struct header **header)
7994 {
7995         struct header *h0 = NULL;
7996         struct field *hf, *mf;
7997         uint32_t i;
7998
7999         /* Return if no match fields. */
8000         if (!params->n_fields || !params->field_names)
8001                 return -EINVAL;
8002
8003         /* Check that all the match fields either belong to the same header
8004          * or are all meta-data fields.
8005          */
8006         hf = header_field_parse(p, params->field_names[0], &h0);
8007         mf = metadata_field_parse(p, params->field_names[0]);
8008         if (!hf && !mf)
8009                 return -EINVAL;
8010
8011         for (i = 1; i < params->n_fields; i++)
8012                 if (h0) {
8013                         struct header *h;
8014
8015                         hf = header_field_parse(p, params->field_names[i], &h);
8016                         if (!hf || (h->id != h0->id))
8017                                 return -EINVAL;
8018                 } else {
8019                         mf = metadata_field_parse(p, params->field_names[i]);
8020                         if (!mf)
8021                                 return -EINVAL;
8022                 }
8023
8024         /* Check that there are no duplicated match fields. */
8025         for (i = 0; i < params->n_fields; i++) {
8026                 const char *field_name = params->field_names[i];
8027                 uint32_t j;
8028
8029                 for (j = i + 1; j < params->n_fields; j++)
8030                         if (!strcmp(params->field_names[j], field_name))
8031                                 return -EINVAL;
8032         }
8033
8034         /* Return. */
8035         if (header)
8036                 *header = h0;
8037
8038         return 0;
8039 }
8040
8041 static int
8042 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8043 {
8044         struct struct_type *mst = p->metadata_st, *ast = a->st;
8045         struct field *mf, *af;
8046         uint32_t mf_pos, i;
8047
8048         if (!ast) {
8049                 if (mf_name)
8050                         return -EINVAL;
8051
8052                 return 0;
8053         }
8054
8055         /* Check that mf_name is the name of a valid meta-data field. */
8056         CHECK_NAME(mf_name, EINVAL);
8057         mf = metadata_field_parse(p, mf_name);
8058         CHECK(mf, EINVAL);
8059
8060         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8061          * all the action arguments.
8062          */
8063         mf_pos = mf - mst->fields;
8064         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8065
8066         /* Check that the size of each of the identified meta-data fields matches exactly the size
8067          * of the corresponding action argument.
8068          */
8069         for (i = 0; i < ast->n_fields; i++) {
8070                 mf = &mst->fields[mf_pos + i];
8071                 af = &ast->fields[i];
8072
8073                 CHECK(mf->n_bits == af->n_bits, EINVAL);
8074         }
8075
8076         return 0;
8077 }
8078
8079 static int
8080 learner_action_learning_check(struct rte_swx_pipeline *p,
8081                               struct action *action,
8082                               const char **action_names,
8083                               uint32_t n_actions)
8084 {
8085         uint32_t i;
8086
8087         /* For each "learn" instruction of the current action, check that the learned action (i.e.
8088          * the action passed as argument to the "learn" instruction) is also enabled for the
8089          * current learner table.
8090          */
8091         for (i = 0; i < action->n_instructions; i++) {
8092                 struct instruction *instr = &action->instructions[i];
8093                 uint32_t found = 0, j;
8094
8095                 if (instr->type != INSTR_LEARNER_LEARN)
8096                         continue;
8097
8098                 for (j = 0; j < n_actions; j++) {
8099                         struct action *a;
8100
8101                         a = action_find(p, action_names[j]);
8102                         if (!a)
8103                                 return -EINVAL;
8104
8105                         if (a->id == instr->learn.action_id)
8106                                 found = 1;
8107                 }
8108
8109                 if (!found)
8110                         return -EINVAL;
8111         }
8112
8113         return 0;
8114 }
8115
8116 int
8117 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8118                               const char *name,
8119                               struct rte_swx_pipeline_learner_params *params,
8120                               uint32_t size,
8121                               uint32_t timeout)
8122 {
8123         struct learner *l = NULL;
8124         struct action *default_action;
8125         struct header *header = NULL;
8126         uint32_t action_data_size_max = 0, i;
8127         int status = 0;
8128
8129         CHECK(p, EINVAL);
8130
8131         CHECK_NAME(name, EINVAL);
8132         CHECK(!table_find(p, name), EEXIST);
8133         CHECK(!selector_find(p, name), EEXIST);
8134         CHECK(!learner_find(p, name), EEXIST);
8135
8136         CHECK(params, EINVAL);
8137
8138         /* Match checks. */
8139         status = learner_match_fields_check(p, params, &header);
8140         if (status)
8141                 return status;
8142
8143         /* Action checks. */
8144         CHECK(params->n_actions, EINVAL);
8145
8146         CHECK(params->action_names, EINVAL);
8147         for (i = 0; i < params->n_actions; i++) {
8148                 const char *action_name = params->action_names[i];
8149                 const char *action_field_name = params->action_field_names[i];
8150                 struct action *a;
8151                 uint32_t action_data_size;
8152
8153                 CHECK_NAME(action_name, EINVAL);
8154
8155                 a = action_find(p, action_name);
8156                 CHECK(a, EINVAL);
8157
8158                 status = learner_action_args_check(p, a, action_field_name);
8159                 if (status)
8160                         return status;
8161
8162                 status = learner_action_learning_check(p,
8163                                                        a,
8164                                                        params->action_names,
8165                                                        params->n_actions);
8166                 if (status)
8167                         return status;
8168
8169                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8170                 if (action_data_size > action_data_size_max)
8171                         action_data_size_max = action_data_size;
8172         }
8173
8174         CHECK_NAME(params->default_action_name, EINVAL);
8175         for (i = 0; i < p->n_actions; i++)
8176                 if (!strcmp(params->action_names[i],
8177                             params->default_action_name))
8178                         break;
8179         CHECK(i < params->n_actions, EINVAL);
8180
8181         default_action = action_find(p, params->default_action_name);
8182         CHECK((default_action->st && params->default_action_data) ||
8183               !params->default_action_data, EINVAL);
8184
8185         /* Any other checks. */
8186         CHECK(size, EINVAL);
8187         CHECK(timeout, EINVAL);
8188
8189         /* Memory allocation. */
8190         l = calloc(1, sizeof(struct learner));
8191         if (!l)
8192                 goto nomem;
8193
8194         l->fields = calloc(params->n_fields, sizeof(struct field *));
8195         if (!l->fields)
8196                 goto nomem;
8197
8198         l->actions = calloc(params->n_actions, sizeof(struct action *));
8199         if (!l->actions)
8200                 goto nomem;
8201
8202         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
8203         if (!l->action_arg)
8204                 goto nomem;
8205
8206         if (action_data_size_max) {
8207                 l->default_action_data = calloc(1, action_data_size_max);
8208                 if (!l->default_action_data)
8209                         goto nomem;
8210         }
8211
8212         /* Node initialization. */
8213         strcpy(l->name, name);
8214
8215         for (i = 0; i < params->n_fields; i++) {
8216                 const char *field_name = params->field_names[i];
8217
8218                 l->fields[i] = header ?
8219                         header_field_parse(p, field_name, NULL) :
8220                         metadata_field_parse(p, field_name);
8221         }
8222
8223         l->n_fields = params->n_fields;
8224
8225         l->header = header;
8226
8227         for (i = 0; i < params->n_actions; i++) {
8228                 const char *mf_name = params->action_field_names[i];
8229
8230                 l->actions[i] = action_find(p, params->action_names[i]);
8231
8232                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
8233         }
8234
8235         l->default_action = default_action;
8236
8237         if (default_action->st)
8238                 memcpy(l->default_action_data,
8239                        params->default_action_data,
8240                        default_action->st->n_bits / 8);
8241
8242         l->n_actions = params->n_actions;
8243
8244         l->default_action_is_const = params->default_action_is_const;
8245
8246         l->action_data_size_max = action_data_size_max;
8247
8248         l->size = size;
8249
8250         l->timeout = timeout;
8251
8252         l->id = p->n_learners;
8253
8254         /* Node add to tailq. */
8255         TAILQ_INSERT_TAIL(&p->learners, l, node);
8256         p->n_learners++;
8257
8258         return 0;
8259
8260 nomem:
8261         if (!l)
8262                 return -ENOMEM;
8263
8264         free(l->action_arg);
8265         free(l->actions);
8266         free(l->fields);
8267         free(l);
8268
8269         return -ENOMEM;
8270 }
8271
8272 static void
8273 learner_params_free(struct rte_swx_table_learner_params *params)
8274 {
8275         if (!params)
8276                 return;
8277
8278         free(params->key_mask0);
8279
8280         free(params);
8281 }
8282
8283 static struct rte_swx_table_learner_params *
8284 learner_params_get(struct learner *l)
8285 {
8286         struct rte_swx_table_learner_params *params = NULL;
8287         struct field *first, *last;
8288         uint32_t i;
8289
8290         /* Memory allocation. */
8291         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8292         if (!params)
8293                 goto error;
8294
8295         /* Find first (smallest offset) and last (biggest offset) match fields. */
8296         first = l->fields[0];
8297         last = l->fields[0];
8298
8299         for (i = 0; i < l->n_fields; i++) {
8300                 struct field *f = l->fields[i];
8301
8302                 if (f->offset < first->offset)
8303                         first = f;
8304
8305                 if (f->offset > last->offset)
8306                         last = f;
8307         }
8308
8309         /* Key offset and size. */
8310         params->key_offset = first->offset / 8;
8311         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8312
8313         /* Memory allocation. */
8314         params->key_mask0 = calloc(1, params->key_size);
8315         if (!params->key_mask0)
8316                 goto error;
8317
8318         /* Key mask. */
8319         for (i = 0; i < l->n_fields; i++) {
8320                 struct field *f = l->fields[i];
8321                 uint32_t start = (f->offset - first->offset) / 8;
8322                 size_t size = f->n_bits / 8;
8323
8324                 memset(&params->key_mask0[start], 0xFF, size);
8325         }
8326
8327         /* Action data size. */
8328         params->action_data_size = l->action_data_size_max;
8329
8330         /* Maximum number of keys. */
8331         params->n_keys_max = l->size;
8332
8333         /* Timeout. */
8334         params->key_timeout = l->timeout;
8335
8336         return params;
8337
8338 error:
8339         learner_params_free(params);
8340         return NULL;
8341 }
8342
8343 static void
8344 learner_build_free(struct rte_swx_pipeline *p)
8345 {
8346         uint32_t i;
8347
8348         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8349                 struct thread *t = &p->threads[i];
8350                 uint32_t j;
8351
8352                 if (!t->learners)
8353                         continue;
8354
8355                 for (j = 0; j < p->n_learners; j++) {
8356                         struct learner_runtime *r = &t->learners[j];
8357
8358                         free(r->mailbox);
8359                         free(r->action_data);
8360                 }
8361
8362                 free(t->learners);
8363                 t->learners = NULL;
8364         }
8365
8366         if (p->learner_stats) {
8367                 for (i = 0; i < p->n_learners; i++)
8368                         free(p->learner_stats[i].n_pkts_action);
8369
8370                 free(p->learner_stats);
8371         }
8372 }
8373
8374 static int
8375 learner_build(struct rte_swx_pipeline *p)
8376 {
8377         uint32_t i;
8378         int status = 0;
8379
8380         /* Per pipeline: learner statistics. */
8381         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8382         CHECK(p->learner_stats, ENOMEM);
8383
8384         for (i = 0; i < p->n_learners; i++) {
8385                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8386                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8387         }
8388
8389         /* Per thread: learner run-time. */
8390         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8391                 struct thread *t = &p->threads[i];
8392                 struct learner *l;
8393
8394                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8395                 if (!t->learners) {
8396                         status = -ENOMEM;
8397                         goto error;
8398                 }
8399
8400                 TAILQ_FOREACH(l, &p->learners, node) {
8401                         struct learner_runtime *r = &t->learners[l->id];
8402                         uint64_t size;
8403                         uint32_t j;
8404
8405                         /* r->mailbox. */
8406                         size = rte_swx_table_learner_mailbox_size_get();
8407                         if (size) {
8408                                 r->mailbox = calloc(1, size);
8409                                 if (!r->mailbox) {
8410                                         status = -ENOMEM;
8411                                         goto error;
8412                                 }
8413                         }
8414
8415                         /* r->key. */
8416                         r->key = l->header ?
8417                                 &t->structs[l->header->struct_id] :
8418                                 &t->structs[p->metadata_struct_id];
8419
8420                         /* r->action_data. */
8421                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
8422                         if (!r->action_data) {
8423                                 status = -ENOMEM;
8424                                 goto error;
8425                         }
8426
8427                         for (j = 0; j < l->n_actions; j++) {
8428                                 struct action *a = l->actions[j];
8429                                 struct field *mf = l->action_arg[j];
8430                                 uint8_t *m = t->structs[p->metadata_struct_id];
8431
8432                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
8433                         }
8434                 }
8435         }
8436
8437         return 0;
8438
8439 error:
8440         learner_build_free(p);
8441         return status;
8442 }
8443
8444 static void
8445 learner_free(struct rte_swx_pipeline *p)
8446 {
8447         learner_build_free(p);
8448
8449         /* Learner tables. */
8450         for ( ; ; ) {
8451                 struct learner *l;
8452
8453                 l = TAILQ_FIRST(&p->learners);
8454                 if (!l)
8455                         break;
8456
8457                 TAILQ_REMOVE(&p->learners, l, node);
8458                 free(l->fields);
8459                 free(l->actions);
8460                 free(l->action_arg);
8461                 free(l->default_action_data);
8462                 free(l);
8463         }
8464 }
8465
8466 /*
8467  * Table state.
8468  */
8469 static int
8470 table_state_build(struct rte_swx_pipeline *p)
8471 {
8472         struct table *table;
8473         struct selector *s;
8474         struct learner *l;
8475
8476         p->table_state = calloc(p->n_tables + p->n_selectors,
8477                                 sizeof(struct rte_swx_table_state));
8478         CHECK(p->table_state, ENOMEM);
8479
8480         TAILQ_FOREACH(table, &p->tables, node) {
8481                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8482
8483                 if (table->type) {
8484                         struct rte_swx_table_params *params;
8485
8486                         /* ts->obj. */
8487                         params = table_params_get(table);
8488                         CHECK(params, ENOMEM);
8489
8490                         ts->obj = table->type->ops.create(params,
8491                                 NULL,
8492                                 table->args,
8493                                 p->numa_node);
8494
8495                         table_params_free(params);
8496                         CHECK(ts->obj, ENODEV);
8497                 }
8498
8499                 /* ts->default_action_data. */
8500                 if (table->action_data_size_max) {
8501                         ts->default_action_data =
8502                                 malloc(table->action_data_size_max);
8503                         CHECK(ts->default_action_data, ENOMEM);
8504
8505                         memcpy(ts->default_action_data,
8506                                table->default_action_data,
8507                                table->action_data_size_max);
8508                 }
8509
8510                 /* ts->default_action_id. */
8511                 ts->default_action_id = table->default_action->id;
8512         }
8513
8514         TAILQ_FOREACH(s, &p->selectors, node) {
8515                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8516                 struct rte_swx_table_selector_params *params;
8517
8518                 /* ts->obj. */
8519                 params = selector_table_params_get(s);
8520                 CHECK(params, ENOMEM);
8521
8522                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8523
8524                 selector_params_free(params);
8525                 CHECK(ts->obj, ENODEV);
8526         }
8527
8528         TAILQ_FOREACH(l, &p->learners, node) {
8529                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8530                         p->n_selectors + l->id];
8531                 struct rte_swx_table_learner_params *params;
8532
8533                 /* ts->obj. */
8534                 params = learner_params_get(l);
8535                 CHECK(params, ENOMEM);
8536
8537                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8538                 learner_params_free(params);
8539                 CHECK(ts->obj, ENODEV);
8540
8541                 /* ts->default_action_data. */
8542                 if (l->action_data_size_max) {
8543                         ts->default_action_data = malloc(l->action_data_size_max);
8544                         CHECK(ts->default_action_data, ENOMEM);
8545
8546                         memcpy(ts->default_action_data,
8547                                l->default_action_data,
8548                                l->action_data_size_max);
8549                 }
8550
8551                 /* ts->default_action_id. */
8552                 ts->default_action_id = l->default_action->id;
8553         }
8554
8555         return 0;
8556 }
8557
8558 static void
8559 table_state_build_free(struct rte_swx_pipeline *p)
8560 {
8561         uint32_t i;
8562
8563         if (!p->table_state)
8564                 return;
8565
8566         for (i = 0; i < p->n_tables; i++) {
8567                 struct rte_swx_table_state *ts = &p->table_state[i];
8568                 struct table *table = table_find_by_id(p, i);
8569
8570                 /* ts->obj. */
8571                 if (table->type && ts->obj)
8572                         table->type->ops.free(ts->obj);
8573
8574                 /* ts->default_action_data. */
8575                 free(ts->default_action_data);
8576         }
8577
8578         for (i = 0; i < p->n_selectors; i++) {
8579                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8580
8581                 /* ts->obj. */
8582                 if (ts->obj)
8583                         rte_swx_table_selector_free(ts->obj);
8584         }
8585
8586         for (i = 0; i < p->n_learners; i++) {
8587                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8588
8589                 /* ts->obj. */
8590                 if (ts->obj)
8591                         rte_swx_table_learner_free(ts->obj);
8592
8593                 /* ts->default_action_data. */
8594                 free(ts->default_action_data);
8595         }
8596
8597         free(p->table_state);
8598         p->table_state = NULL;
8599 }
8600
8601 static void
8602 table_state_free(struct rte_swx_pipeline *p)
8603 {
8604         table_state_build_free(p);
8605 }
8606
8607 /*
8608  * Register array.
8609  */
8610 static struct regarray *
8611 regarray_find(struct rte_swx_pipeline *p, const char *name)
8612 {
8613         struct regarray *elem;
8614
8615         TAILQ_FOREACH(elem, &p->regarrays, node)
8616                 if (!strcmp(elem->name, name))
8617                         return elem;
8618
8619         return NULL;
8620 }
8621
8622 static struct regarray *
8623 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8624 {
8625         struct regarray *elem = NULL;
8626
8627         TAILQ_FOREACH(elem, &p->regarrays, node)
8628                 if (elem->id == id)
8629                         return elem;
8630
8631         return NULL;
8632 }
8633
8634 int
8635 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8636                               const char *name,
8637                               uint32_t size,
8638                               uint64_t init_val)
8639 {
8640         struct regarray *r;
8641
8642         CHECK(p, EINVAL);
8643
8644         CHECK_NAME(name, EINVAL);
8645         CHECK(!regarray_find(p, name), EEXIST);
8646
8647         CHECK(size, EINVAL);
8648         size = rte_align32pow2(size);
8649
8650         /* Memory allocation. */
8651         r = calloc(1, sizeof(struct regarray));
8652         CHECK(r, ENOMEM);
8653
8654         /* Node initialization. */
8655         strcpy(r->name, name);
8656         r->init_val = init_val;
8657         r->size = size;
8658         r->id = p->n_regarrays;
8659
8660         /* Node add to tailq. */
8661         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8662         p->n_regarrays++;
8663
8664         return 0;
8665 }
8666
8667 static int
8668 regarray_build(struct rte_swx_pipeline *p)
8669 {
8670         struct regarray *regarray;
8671
8672         if (!p->n_regarrays)
8673                 return 0;
8674
8675         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8676         CHECK(p->regarray_runtime, ENOMEM);
8677
8678         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8679                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8680                 uint32_t i;
8681
8682                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8683                                          RTE_CACHE_LINE_SIZE,
8684                                          p->numa_node);
8685                 CHECK(r->regarray, ENOMEM);
8686
8687                 if (regarray->init_val)
8688                         for (i = 0; i < regarray->size; i++)
8689                                 r->regarray[i] = regarray->init_val;
8690
8691                 r->size_mask = regarray->size - 1;
8692         }
8693
8694         return 0;
8695 }
8696
8697 static void
8698 regarray_build_free(struct rte_swx_pipeline *p)
8699 {
8700         uint32_t i;
8701
8702         if (!p->regarray_runtime)
8703                 return;
8704
8705         for (i = 0; i < p->n_regarrays; i++) {
8706                 struct regarray *regarray = regarray_find_by_id(p, i);
8707                 struct regarray_runtime *r = &p->regarray_runtime[i];
8708
8709                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8710         }
8711
8712         free(p->regarray_runtime);
8713         p->regarray_runtime = NULL;
8714 }
8715
8716 static void
8717 regarray_free(struct rte_swx_pipeline *p)
8718 {
8719         regarray_build_free(p);
8720
8721         for ( ; ; ) {
8722                 struct regarray *elem;
8723
8724                 elem = TAILQ_FIRST(&p->regarrays);
8725                 if (!elem)
8726                         break;
8727
8728                 TAILQ_REMOVE(&p->regarrays, elem, node);
8729                 free(elem);
8730         }
8731 }
8732
8733 /*
8734  * Meter array.
8735  */
8736 static struct meter_profile *
8737 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8738 {
8739         struct meter_profile *elem;
8740
8741         TAILQ_FOREACH(elem, &p->meter_profiles, node)
8742                 if (!strcmp(elem->name, name))
8743                         return elem;
8744
8745         return NULL;
8746 }
8747
8748 static struct metarray *
8749 metarray_find(struct rte_swx_pipeline *p, const char *name)
8750 {
8751         struct metarray *elem;
8752
8753         TAILQ_FOREACH(elem, &p->metarrays, node)
8754                 if (!strcmp(elem->name, name))
8755                         return elem;
8756
8757         return NULL;
8758 }
8759
8760 static struct metarray *
8761 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8762 {
8763         struct metarray *elem = NULL;
8764
8765         TAILQ_FOREACH(elem, &p->metarrays, node)
8766                 if (elem->id == id)
8767                         return elem;
8768
8769         return NULL;
8770 }
8771
8772 int
8773 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8774                                  const char *name,
8775                                  uint32_t size)
8776 {
8777         struct metarray *m;
8778
8779         CHECK(p, EINVAL);
8780
8781         CHECK_NAME(name, EINVAL);
8782         CHECK(!metarray_find(p, name), EEXIST);
8783
8784         CHECK(size, EINVAL);
8785         size = rte_align32pow2(size);
8786
8787         /* Memory allocation. */
8788         m = calloc(1, sizeof(struct metarray));
8789         CHECK(m, ENOMEM);
8790
8791         /* Node initialization. */
8792         strcpy(m->name, name);
8793         m->size = size;
8794         m->id = p->n_metarrays;
8795
8796         /* Node add to tailq. */
8797         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8798         p->n_metarrays++;
8799
8800         return 0;
8801 }
8802
8803 struct meter_profile meter_profile_default = {
8804         .node = {0},
8805         .name = "",
8806         .params = {0},
8807
8808         .profile = {
8809                 .cbs = 10000,
8810                 .pbs = 10000,
8811                 .cir_period = 1,
8812                 .cir_bytes_per_period = 1,
8813                 .pir_period = 1,
8814                 .pir_bytes_per_period = 1,
8815         },
8816
8817         .n_users = 0,
8818 };
8819
8820 static void
8821 meter_init(struct meter *m)
8822 {
8823         memset(m, 0, sizeof(struct meter));
8824         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8825         m->profile = &meter_profile_default;
8826         m->color_mask = RTE_COLOR_GREEN;
8827
8828         meter_profile_default.n_users++;
8829 }
8830
8831 static int
8832 metarray_build(struct rte_swx_pipeline *p)
8833 {
8834         struct metarray *m;
8835
8836         if (!p->n_metarrays)
8837                 return 0;
8838
8839         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8840         CHECK(p->metarray_runtime, ENOMEM);
8841
8842         TAILQ_FOREACH(m, &p->metarrays, node) {
8843                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8844                 uint32_t i;
8845
8846                 r->metarray = env_malloc(m->size * sizeof(struct meter),
8847                                          RTE_CACHE_LINE_SIZE,
8848                                          p->numa_node);
8849                 CHECK(r->metarray, ENOMEM);
8850
8851                 for (i = 0; i < m->size; i++)
8852                         meter_init(&r->metarray[i]);
8853
8854                 r->size_mask = m->size - 1;
8855         }
8856
8857         return 0;
8858 }
8859
8860 static void
8861 metarray_build_free(struct rte_swx_pipeline *p)
8862 {
8863         uint32_t i;
8864
8865         if (!p->metarray_runtime)
8866                 return;
8867
8868         for (i = 0; i < p->n_metarrays; i++) {
8869                 struct metarray *m = metarray_find_by_id(p, i);
8870                 struct metarray_runtime *r = &p->metarray_runtime[i];
8871
8872                 env_free(r->metarray, m->size * sizeof(struct meter));
8873         }
8874
8875         free(p->metarray_runtime);
8876         p->metarray_runtime = NULL;
8877 }
8878
8879 static void
8880 metarray_free(struct rte_swx_pipeline *p)
8881 {
8882         metarray_build_free(p);
8883
8884         /* Meter arrays. */
8885         for ( ; ; ) {
8886                 struct metarray *elem;
8887
8888                 elem = TAILQ_FIRST(&p->metarrays);
8889                 if (!elem)
8890                         break;
8891
8892                 TAILQ_REMOVE(&p->metarrays, elem, node);
8893                 free(elem);
8894         }
8895
8896         /* Meter profiles. */
8897         for ( ; ; ) {
8898                 struct meter_profile *elem;
8899
8900                 elem = TAILQ_FIRST(&p->meter_profiles);
8901                 if (!elem)
8902                         break;
8903
8904                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8905                 free(elem);
8906         }
8907 }
8908
8909 /*
8910  * Pipeline.
8911  */
8912 int
8913 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8914 {
8915         struct rte_swx_pipeline *pipeline;
8916
8917         /* Check input parameters. */
8918         CHECK(p, EINVAL);
8919
8920         /* Memory allocation. */
8921         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8922         CHECK(pipeline, ENOMEM);
8923
8924         /* Initialization. */
8925         TAILQ_INIT(&pipeline->struct_types);
8926         TAILQ_INIT(&pipeline->port_in_types);
8927         TAILQ_INIT(&pipeline->ports_in);
8928         TAILQ_INIT(&pipeline->port_out_types);
8929         TAILQ_INIT(&pipeline->ports_out);
8930         TAILQ_INIT(&pipeline->extern_types);
8931         TAILQ_INIT(&pipeline->extern_objs);
8932         TAILQ_INIT(&pipeline->extern_funcs);
8933         TAILQ_INIT(&pipeline->headers);
8934         TAILQ_INIT(&pipeline->actions);
8935         TAILQ_INIT(&pipeline->table_types);
8936         TAILQ_INIT(&pipeline->tables);
8937         TAILQ_INIT(&pipeline->selectors);
8938         TAILQ_INIT(&pipeline->learners);
8939         TAILQ_INIT(&pipeline->regarrays);
8940         TAILQ_INIT(&pipeline->meter_profiles);
8941         TAILQ_INIT(&pipeline->metarrays);
8942
8943         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
8944         pipeline->numa_node = numa_node;
8945
8946         *p = pipeline;
8947         return 0;
8948 }
8949
8950 void
8951 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8952 {
8953         if (!p)
8954                 return;
8955
8956         free(p->instruction_data);
8957         free(p->instructions);
8958
8959         metarray_free(p);
8960         regarray_free(p);
8961         table_state_free(p);
8962         learner_free(p);
8963         selector_free(p);
8964         table_free(p);
8965         action_free(p);
8966         instruction_table_free(p);
8967         metadata_free(p);
8968         header_free(p);
8969         extern_func_free(p);
8970         extern_obj_free(p);
8971         port_out_free(p);
8972         port_in_free(p);
8973         struct_free(p);
8974
8975         free(p);
8976 }
8977
8978 int
8979 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
8980                                      const char **instructions,
8981                                      uint32_t n_instructions)
8982 {
8983         int err;
8984         uint32_t i;
8985
8986         err = instruction_config(p, NULL, instructions, n_instructions);
8987         if (err)
8988                 return err;
8989
8990         /* Thread instruction pointer reset. */
8991         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8992                 struct thread *t = &p->threads[i];
8993
8994                 thread_ip_reset(p, t);
8995         }
8996
8997         return 0;
8998 }
8999
9000 static int
9001 pipeline_compile(struct rte_swx_pipeline *p);
9002
9003 int
9004 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9005 {
9006         int status;
9007
9008         CHECK(p, EINVAL);
9009         CHECK(p->build_done == 0, EEXIST);
9010
9011         status = port_in_build(p);
9012         if (status)
9013                 goto error;
9014
9015         status = port_out_build(p);
9016         if (status)
9017                 goto error;
9018
9019         status = struct_build(p);
9020         if (status)
9021                 goto error;
9022
9023         status = extern_obj_build(p);
9024         if (status)
9025                 goto error;
9026
9027         status = extern_func_build(p);
9028         if (status)
9029                 goto error;
9030
9031         status = header_build(p);
9032         if (status)
9033                 goto error;
9034
9035         status = metadata_build(p);
9036         if (status)
9037                 goto error;
9038
9039         status = instruction_table_build(p);
9040         if (status)
9041                 goto error;
9042
9043         status = action_build(p);
9044         if (status)
9045                 goto error;
9046
9047         status = table_build(p);
9048         if (status)
9049                 goto error;
9050
9051         status = selector_build(p);
9052         if (status)
9053                 goto error;
9054
9055         status = learner_build(p);
9056         if (status)
9057                 goto error;
9058
9059         status = table_state_build(p);
9060         if (status)
9061                 goto error;
9062
9063         status = regarray_build(p);
9064         if (status)
9065                 goto error;
9066
9067         status = metarray_build(p);
9068         if (status)
9069                 goto error;
9070
9071         p->build_done = 1;
9072
9073         pipeline_compile(p);
9074
9075         return 0;
9076
9077 error:
9078         metarray_build_free(p);
9079         regarray_build_free(p);
9080         table_state_build_free(p);
9081         learner_build_free(p);
9082         selector_build_free(p);
9083         table_build_free(p);
9084         action_build_free(p);
9085         instruction_table_build_free(p);
9086         metadata_build_free(p);
9087         header_build_free(p);
9088         extern_func_build_free(p);
9089         extern_obj_build_free(p);
9090         port_out_build_free(p);
9091         port_in_build_free(p);
9092         struct_build_free(p);
9093
9094         return status;
9095 }
9096
9097 void
9098 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9099 {
9100         uint32_t i;
9101
9102         for (i = 0; i < n_instructions; i++)
9103                 instr_exec(p);
9104 }
9105
9106 void
9107 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9108 {
9109         uint32_t i;
9110
9111         for (i = 0; i < p->n_ports_out; i++) {
9112                 struct port_out_runtime *port = &p->out[i];
9113
9114                 if (port->flush)
9115                         port->flush(port->obj);
9116         }
9117 }
9118
9119 /*
9120  * Control.
9121  */
9122 int
9123 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9124                               struct rte_swx_ctl_pipeline_info *pipeline)
9125 {
9126         struct action *action;
9127         struct table *table;
9128         uint32_t n_actions = 0, n_tables = 0;
9129
9130         if (!p || !pipeline)
9131                 return -EINVAL;
9132
9133         TAILQ_FOREACH(action, &p->actions, node)
9134                 n_actions++;
9135
9136         TAILQ_FOREACH(table, &p->tables, node)
9137                 n_tables++;
9138
9139         pipeline->n_ports_in = p->n_ports_in;
9140         pipeline->n_ports_out = p->n_ports_out;
9141         pipeline->n_actions = n_actions;
9142         pipeline->n_tables = n_tables;
9143         pipeline->n_selectors = p->n_selectors;
9144         pipeline->n_learners = p->n_learners;
9145         pipeline->n_regarrays = p->n_regarrays;
9146         pipeline->n_metarrays = p->n_metarrays;
9147
9148         return 0;
9149 }
9150
9151 int
9152 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9153 {
9154         if (!p || !numa_node)
9155                 return -EINVAL;
9156
9157         *numa_node = p->numa_node;
9158         return 0;
9159 }
9160
9161 int
9162 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9163                             uint32_t action_id,
9164                             struct rte_swx_ctl_action_info *action)
9165 {
9166         struct action *a = NULL;
9167
9168         if (!p || (action_id >= p->n_actions) || !action)
9169                 return -EINVAL;
9170
9171         a = action_find_by_id(p, action_id);
9172         if (!a)
9173                 return -EINVAL;
9174
9175         strcpy(action->name, a->name);
9176         action->n_args = a->st ? a->st->n_fields : 0;
9177         return 0;
9178 }
9179
9180 int
9181 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9182                                 uint32_t action_id,
9183                                 uint32_t action_arg_id,
9184                                 struct rte_swx_ctl_action_arg_info *action_arg)
9185 {
9186         struct action *a = NULL;
9187         struct field *arg = NULL;
9188
9189         if (!p || (action_id >= p->n_actions) || !action_arg)
9190                 return -EINVAL;
9191
9192         a = action_find_by_id(p, action_id);
9193         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9194                 return -EINVAL;
9195
9196         arg = &a->st->fields[action_arg_id];
9197         strcpy(action_arg->name, arg->name);
9198         action_arg->n_bits = arg->n_bits;
9199         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9200
9201         return 0;
9202 }
9203
9204 int
9205 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9206                            uint32_t table_id,
9207                            struct rte_swx_ctl_table_info *table)
9208 {
9209         struct table *t = NULL;
9210
9211         if (!p || !table)
9212                 return -EINVAL;
9213
9214         t = table_find_by_id(p, table_id);
9215         if (!t)
9216                 return -EINVAL;
9217
9218         strcpy(table->name, t->name);
9219         strcpy(table->args, t->args);
9220         table->n_match_fields = t->n_fields;
9221         table->n_actions = t->n_actions;
9222         table->default_action_is_const = t->default_action_is_const;
9223         table->size = t->size;
9224         return 0;
9225 }
9226
9227 int
9228 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9229         uint32_t table_id,
9230         uint32_t match_field_id,
9231         struct rte_swx_ctl_table_match_field_info *match_field)
9232 {
9233         struct table *t;
9234         struct match_field *f;
9235
9236         if (!p || (table_id >= p->n_tables) || !match_field)
9237                 return -EINVAL;
9238
9239         t = table_find_by_id(p, table_id);
9240         if (!t || (match_field_id >= t->n_fields))
9241                 return -EINVAL;
9242
9243         f = &t->fields[match_field_id];
9244         match_field->match_type = f->match_type;
9245         match_field->is_header = t->header ? 1 : 0;
9246         match_field->n_bits = f->field->n_bits;
9247         match_field->offset = f->field->offset;
9248
9249         return 0;
9250 }
9251
9252 int
9253 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9254         uint32_t table_id,
9255         uint32_t table_action_id,
9256         struct rte_swx_ctl_table_action_info *table_action)
9257 {
9258         struct table *t;
9259
9260         if (!p || (table_id >= p->n_tables) || !table_action)
9261                 return -EINVAL;
9262
9263         t = table_find_by_id(p, table_id);
9264         if (!t || (table_action_id >= t->n_actions))
9265                 return -EINVAL;
9266
9267         table_action->action_id = t->actions[table_action_id]->id;
9268
9269         return 0;
9270 }
9271
9272 int
9273 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9274                           uint32_t table_id,
9275                           struct rte_swx_table_ops *table_ops,
9276                           int *is_stub)
9277 {
9278         struct table *t;
9279
9280         if (!p || (table_id >= p->n_tables))
9281                 return -EINVAL;
9282
9283         t = table_find_by_id(p, table_id);
9284         if (!t)
9285                 return -EINVAL;
9286
9287         if (t->type) {
9288                 if (table_ops)
9289                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9290                 *is_stub = 0;
9291         } else {
9292                 *is_stub = 1;
9293         }
9294
9295         return 0;
9296 }
9297
9298 int
9299 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9300                               uint32_t selector_id,
9301                               struct rte_swx_ctl_selector_info *selector)
9302 {
9303         struct selector *s = NULL;
9304
9305         if (!p || !selector)
9306                 return -EINVAL;
9307
9308         s = selector_find_by_id(p, selector_id);
9309         if (!s)
9310                 return -EINVAL;
9311
9312         strcpy(selector->name, s->name);
9313
9314         selector->n_selector_fields = s->n_selector_fields;
9315         selector->n_groups_max = s->n_groups_max;
9316         selector->n_members_per_group_max = s->n_members_per_group_max;
9317
9318         return 0;
9319 }
9320
9321 int
9322 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9323          uint32_t selector_id,
9324          struct rte_swx_ctl_table_match_field_info *field)
9325 {
9326         struct selector *s;
9327
9328         if (!p || (selector_id >= p->n_selectors) || !field)
9329                 return -EINVAL;
9330
9331         s = selector_find_by_id(p, selector_id);
9332         if (!s)
9333                 return -EINVAL;
9334
9335         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9336         field->is_header = 0;
9337         field->n_bits = s->group_id_field->n_bits;
9338         field->offset = s->group_id_field->offset;
9339
9340         return 0;
9341 }
9342
9343 int
9344 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9345          uint32_t selector_id,
9346          uint32_t selector_field_id,
9347          struct rte_swx_ctl_table_match_field_info *field)
9348 {
9349         struct selector *s;
9350         struct field *f;
9351
9352         if (!p || (selector_id >= p->n_selectors) || !field)
9353                 return -EINVAL;
9354
9355         s = selector_find_by_id(p, selector_id);
9356         if (!s || (selector_field_id >= s->n_selector_fields))
9357                 return -EINVAL;
9358
9359         f = s->selector_fields[selector_field_id];
9360         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9361         field->is_header = s->selector_header ? 1 : 0;
9362         field->n_bits = f->n_bits;
9363         field->offset = f->offset;
9364
9365         return 0;
9366 }
9367
9368 int
9369 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9370          uint32_t selector_id,
9371          struct rte_swx_ctl_table_match_field_info *field)
9372 {
9373         struct selector *s;
9374
9375         if (!p || (selector_id >= p->n_selectors) || !field)
9376                 return -EINVAL;
9377
9378         s = selector_find_by_id(p, selector_id);
9379         if (!s)
9380                 return -EINVAL;
9381
9382         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9383         field->is_header = 0;
9384         field->n_bits = s->member_id_field->n_bits;
9385         field->offset = s->member_id_field->offset;
9386
9387         return 0;
9388 }
9389
9390 int
9391 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9392                              uint32_t learner_id,
9393                              struct rte_swx_ctl_learner_info *learner)
9394 {
9395         struct learner *l = NULL;
9396
9397         if (!p || !learner)
9398                 return -EINVAL;
9399
9400         l = learner_find_by_id(p, learner_id);
9401         if (!l)
9402                 return -EINVAL;
9403
9404         strcpy(learner->name, l->name);
9405
9406         learner->n_match_fields = l->n_fields;
9407         learner->n_actions = l->n_actions;
9408         learner->default_action_is_const = l->default_action_is_const;
9409         learner->size = l->size;
9410
9411         return 0;
9412 }
9413
9414 int
9415 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9416                                          uint32_t learner_id,
9417                                          uint32_t match_field_id,
9418                                          struct rte_swx_ctl_table_match_field_info *match_field)
9419 {
9420         struct learner *l;
9421         struct field *f;
9422
9423         if (!p || (learner_id >= p->n_learners) || !match_field)
9424                 return -EINVAL;
9425
9426         l = learner_find_by_id(p, learner_id);
9427         if (!l || (match_field_id >= l->n_fields))
9428                 return -EINVAL;
9429
9430         f = l->fields[match_field_id];
9431         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9432         match_field->is_header = l->header ? 1 : 0;
9433         match_field->n_bits = f->n_bits;
9434         match_field->offset = f->offset;
9435
9436         return 0;
9437 }
9438
9439 int
9440 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9441                                     uint32_t learner_id,
9442                                     uint32_t learner_action_id,
9443                                     struct rte_swx_ctl_table_action_info *learner_action)
9444 {
9445         struct learner *l;
9446
9447         if (!p || (learner_id >= p->n_learners) || !learner_action)
9448                 return -EINVAL;
9449
9450         l = learner_find_by_id(p, learner_id);
9451         if (!l || (learner_action_id >= l->n_actions))
9452                 return -EINVAL;
9453
9454         learner_action->action_id = l->actions[learner_action_id]->id;
9455
9456         return 0;
9457 }
9458
9459 int
9460 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9461                                  struct rte_swx_table_state **table_state)
9462 {
9463         if (!p || !table_state || !p->build_done)
9464                 return -EINVAL;
9465
9466         *table_state = p->table_state;
9467         return 0;
9468 }
9469
9470 int
9471 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9472                                  struct rte_swx_table_state *table_state)
9473 {
9474         if (!p || !table_state || !p->build_done)
9475                 return -EINVAL;
9476
9477         p->table_state = table_state;
9478         return 0;
9479 }
9480
9481 int
9482 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9483                                         uint32_t port_id,
9484                                         struct rte_swx_port_in_stats *stats)
9485 {
9486         struct port_in *port;
9487
9488         if (!p || !stats)
9489                 return -EINVAL;
9490
9491         port = port_in_find(p, port_id);
9492         if (!port)
9493                 return -EINVAL;
9494
9495         port->type->ops.stats_read(port->obj, stats);
9496         return 0;
9497 }
9498
9499 int
9500 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9501                                          uint32_t port_id,
9502                                          struct rte_swx_port_out_stats *stats)
9503 {
9504         struct port_out *port;
9505
9506         if (!p || !stats)
9507                 return -EINVAL;
9508
9509         port = port_out_find(p, port_id);
9510         if (!port)
9511                 return -EINVAL;
9512
9513         port->type->ops.stats_read(port->obj, stats);
9514         return 0;
9515 }
9516
9517 int
9518 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9519                                       const char *table_name,
9520                                       struct rte_swx_table_stats *stats)
9521 {
9522         struct table *table;
9523         struct table_statistics *table_stats;
9524
9525         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9526                 return -EINVAL;
9527
9528         table = table_find(p, table_name);
9529         if (!table)
9530                 return -EINVAL;
9531
9532         table_stats = &p->table_stats[table->id];
9533
9534         memcpy(stats->n_pkts_action,
9535                table_stats->n_pkts_action,
9536                p->n_actions * sizeof(uint64_t));
9537
9538         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9539         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9540
9541         return 0;
9542 }
9543
9544 int
9545 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9546         const char *selector_name,
9547         struct rte_swx_pipeline_selector_stats *stats)
9548 {
9549         struct selector *s;
9550
9551         if (!p || !selector_name || !selector_name[0] || !stats)
9552                 return -EINVAL;
9553
9554         s = selector_find(p, selector_name);
9555         if (!s)
9556                 return -EINVAL;
9557
9558         stats->n_pkts = p->selector_stats[s->id].n_pkts;
9559
9560         return 0;
9561 }
9562
9563 int
9564 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9565                                         const char *learner_name,
9566                                         struct rte_swx_learner_stats *stats)
9567 {
9568         struct learner *l;
9569         struct learner_statistics *learner_stats;
9570
9571         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9572                 return -EINVAL;
9573
9574         l = learner_find(p, learner_name);
9575         if (!l)
9576                 return -EINVAL;
9577
9578         learner_stats = &p->learner_stats[l->id];
9579
9580         memcpy(stats->n_pkts_action,
9581                learner_stats->n_pkts_action,
9582                p->n_actions * sizeof(uint64_t));
9583
9584         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9585         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9586
9587         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9588         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9589
9590         stats->n_pkts_forget = learner_stats->n_pkts_forget;
9591
9592         return 0;
9593 }
9594
9595 int
9596 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9597                               uint32_t regarray_id,
9598                               struct rte_swx_ctl_regarray_info *regarray)
9599 {
9600         struct regarray *r;
9601
9602         if (!p || !regarray)
9603                 return -EINVAL;
9604
9605         r = regarray_find_by_id(p, regarray_id);
9606         if (!r)
9607                 return -EINVAL;
9608
9609         strcpy(regarray->name, r->name);
9610         regarray->size = r->size;
9611         return 0;
9612 }
9613
9614 int
9615 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9616                                    const char *regarray_name,
9617                                    uint32_t regarray_index,
9618                                    uint64_t *value)
9619 {
9620         struct regarray *regarray;
9621         struct regarray_runtime *r;
9622
9623         if (!p || !regarray_name || !value)
9624                 return -EINVAL;
9625
9626         regarray = regarray_find(p, regarray_name);
9627         if (!regarray || (regarray_index >= regarray->size))
9628                 return -EINVAL;
9629
9630         r = &p->regarray_runtime[regarray->id];
9631         *value = r->regarray[regarray_index];
9632         return 0;
9633 }
9634
9635 int
9636 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9637                                    const char *regarray_name,
9638                                    uint32_t regarray_index,
9639                                    uint64_t value)
9640 {
9641         struct regarray *regarray;
9642         struct regarray_runtime *r;
9643
9644         if (!p || !regarray_name)
9645                 return -EINVAL;
9646
9647         regarray = regarray_find(p, regarray_name);
9648         if (!regarray || (regarray_index >= regarray->size))
9649                 return -EINVAL;
9650
9651         r = &p->regarray_runtime[regarray->id];
9652         r->regarray[regarray_index] = value;
9653         return 0;
9654 }
9655
9656 int
9657 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9658                               uint32_t metarray_id,
9659                               struct rte_swx_ctl_metarray_info *metarray)
9660 {
9661         struct metarray *m;
9662
9663         if (!p || !metarray)
9664                 return -EINVAL;
9665
9666         m = metarray_find_by_id(p, metarray_id);
9667         if (!m)
9668                 return -EINVAL;
9669
9670         strcpy(metarray->name, m->name);
9671         metarray->size = m->size;
9672         return 0;
9673 }
9674
9675 int
9676 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9677                               const char *name,
9678                               struct rte_meter_trtcm_params *params)
9679 {
9680         struct meter_profile *mp;
9681         int status;
9682
9683         CHECK(p, EINVAL);
9684         CHECK_NAME(name, EINVAL);
9685         CHECK(params, EINVAL);
9686         CHECK(!meter_profile_find(p, name), EEXIST);
9687
9688         /* Node allocation. */
9689         mp = calloc(1, sizeof(struct meter_profile));
9690         CHECK(mp, ENOMEM);
9691
9692         /* Node initialization. */
9693         strcpy(mp->name, name);
9694         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9695         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9696         if (status) {
9697                 free(mp);
9698                 CHECK(0, EINVAL);
9699         }
9700
9701         /* Node add to tailq. */
9702         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9703
9704         return 0;
9705 }
9706
9707 int
9708 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9709                                  const char *name)
9710 {
9711         struct meter_profile *mp;
9712
9713         CHECK(p, EINVAL);
9714         CHECK_NAME(name, EINVAL);
9715
9716         mp = meter_profile_find(p, name);
9717         CHECK(mp, EINVAL);
9718         CHECK(!mp->n_users, EBUSY);
9719
9720         /* Remove node from tailq. */
9721         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9722         free(mp);
9723
9724         return 0;
9725 }
9726
9727 int
9728 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9729                         const char *metarray_name,
9730                         uint32_t metarray_index)
9731 {
9732         struct meter_profile *mp_old;
9733         struct metarray *metarray;
9734         struct metarray_runtime *metarray_runtime;
9735         struct meter *m;
9736
9737         CHECK(p, EINVAL);
9738         CHECK_NAME(metarray_name, EINVAL);
9739
9740         metarray = metarray_find(p, metarray_name);
9741         CHECK(metarray, EINVAL);
9742         CHECK(metarray_index < metarray->size, EINVAL);
9743
9744         metarray_runtime = &p->metarray_runtime[metarray->id];
9745         m = &metarray_runtime->metarray[metarray_index];
9746         mp_old = m->profile;
9747
9748         meter_init(m);
9749
9750         mp_old->n_users--;
9751
9752         return 0;
9753 }
9754
9755 int
9756 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9757                       const char *metarray_name,
9758                       uint32_t metarray_index,
9759                       const char *profile_name)
9760 {
9761         struct meter_profile *mp, *mp_old;
9762         struct metarray *metarray;
9763         struct metarray_runtime *metarray_runtime;
9764         struct meter *m;
9765
9766         CHECK(p, EINVAL);
9767         CHECK_NAME(metarray_name, EINVAL);
9768
9769         metarray = metarray_find(p, metarray_name);
9770         CHECK(metarray, EINVAL);
9771         CHECK(metarray_index < metarray->size, EINVAL);
9772
9773         mp = meter_profile_find(p, profile_name);
9774         CHECK(mp, EINVAL);
9775
9776         metarray_runtime = &p->metarray_runtime[metarray->id];
9777         m = &metarray_runtime->metarray[metarray_index];
9778         mp_old = m->profile;
9779
9780         memset(m, 0, sizeof(struct meter));
9781         rte_meter_trtcm_config(&m->m, &mp->profile);
9782         m->profile = mp;
9783         m->color_mask = RTE_COLORS;
9784
9785         mp->n_users++;
9786         mp_old->n_users--;
9787
9788         return 0;
9789 }
9790
9791 int
9792 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9793                              const char *metarray_name,
9794                              uint32_t metarray_index,
9795                              struct rte_swx_ctl_meter_stats *stats)
9796 {
9797         struct metarray *metarray;
9798         struct metarray_runtime *metarray_runtime;
9799         struct meter *m;
9800
9801         CHECK(p, EINVAL);
9802         CHECK_NAME(metarray_name, EINVAL);
9803
9804         metarray = metarray_find(p, metarray_name);
9805         CHECK(metarray, EINVAL);
9806         CHECK(metarray_index < metarray->size, EINVAL);
9807
9808         CHECK(stats, EINVAL);
9809
9810         metarray_runtime = &p->metarray_runtime[metarray->id];
9811         m = &metarray_runtime->metarray[metarray_index];
9812
9813         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9814         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9815
9816         return 0;
9817 }
9818
9819 /*
9820  * Pipeline compilation.
9821  */
9822 static const char *
9823 instr_type_to_name(struct instruction *instr)
9824 {
9825         switch (instr->type) {
9826         case INSTR_RX: return "INSTR_RX";
9827
9828         case INSTR_TX: return "INSTR_TX";
9829         case INSTR_TX_I: return "INSTR_TX_I";
9830
9831         case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
9832         case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
9833         case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
9834         case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
9835         case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
9836         case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
9837         case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
9838         case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
9839
9840         case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
9841
9842         case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
9843
9844         case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
9845         case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
9846         case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
9847         case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
9848         case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
9849         case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
9850         case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
9851         case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
9852         case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
9853
9854         case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
9855         case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
9856
9857         case INSTR_MOV: return "INSTR_MOV";
9858         case INSTR_MOV_MH: return "INSTR_MOV_MH";
9859         case INSTR_MOV_HM: return "INSTR_MOV_HM";
9860         case INSTR_MOV_HH: return "INSTR_MOV_HH";
9861         case INSTR_MOV_I: return "INSTR_MOV_I";
9862
9863         case INSTR_DMA_HT: return "INSTR_DMA_HT";
9864         case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
9865         case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
9866         case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
9867         case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
9868         case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
9869         case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
9870         case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
9871
9872         case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
9873         case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
9874         case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
9875         case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
9876         case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
9877         case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
9878
9879         case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
9880         case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
9881         case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
9882         case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
9883         case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
9884         case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
9885
9886         case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
9887         case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
9888         case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
9889         case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
9890
9891         case INSTR_ALU_AND: return "INSTR_ALU_AND";
9892         case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
9893         case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
9894         case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
9895         case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
9896
9897         case INSTR_ALU_OR: return "INSTR_ALU_OR";
9898         case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
9899         case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
9900         case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
9901         case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
9902
9903         case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
9904         case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
9905         case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
9906         case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
9907         case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
9908
9909         case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
9910         case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
9911         case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
9912         case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
9913         case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
9914         case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
9915
9916         case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
9917         case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
9918         case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
9919         case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
9920         case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
9921         case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
9922
9923         case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
9924         case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
9925         case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
9926
9927         case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
9928         case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
9929         case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
9930         case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
9931         case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
9932         case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
9933
9934         case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
9935         case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
9936         case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
9937         case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
9938         case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
9939         case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
9940         case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
9941         case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
9942         case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
9943
9944         case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
9945         case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
9946         case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
9947         case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
9948         case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
9949         case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
9950         case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
9951         case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
9952         case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
9953
9954         case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
9955         case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
9956         case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
9957
9958         case INSTR_METER_HHM: return "INSTR_METER_HHM";
9959         case INSTR_METER_HHI: return "INSTR_METER_HHI";
9960         case INSTR_METER_HMM: return "INSTR_METER_HMM";
9961         case INSTR_METER_HMI: return "INSTR_METER_HMI";
9962         case INSTR_METER_MHM: return "INSTR_METER_MHM";
9963         case INSTR_METER_MHI: return "INSTR_METER_MHI";
9964         case INSTR_METER_MMM: return "INSTR_METER_MMM";
9965         case INSTR_METER_MMI: return "INSTR_METER_MMI";
9966         case INSTR_METER_IHM: return "INSTR_METER_IHM";
9967         case INSTR_METER_IHI: return "INSTR_METER_IHI";
9968         case INSTR_METER_IMM: return "INSTR_METER_IMM";
9969         case INSTR_METER_IMI: return "INSTR_METER_IMI";
9970
9971         case INSTR_TABLE: return "INSTR_TABLE";
9972         case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
9973         case INSTR_SELECTOR: return "INSTR_SELECTOR";
9974         case INSTR_LEARNER: return "INSTR_LEARNER";
9975         case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
9976
9977         case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
9978         case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
9979
9980         case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
9981         case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
9982
9983         case INSTR_JMP: return "INSTR_JMP";
9984         case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
9985         case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
9986         case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
9987         case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
9988         case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
9989         case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
9990         case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
9991         case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
9992         case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
9993         case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
9994         case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
9995         case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
9996         case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
9997         case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
9998         case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
9999         case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10000         case INSTR_JMP_LT: return "INSTR_JMP_LT";
10001         case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10002         case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10003         case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10004         case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10005         case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10006         case INSTR_JMP_GT: return "INSTR_JMP_GT";
10007         case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10008         case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10009         case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10010         case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10011         case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10012
10013         case INSTR_RETURN: return "INSTR_RETURN";
10014
10015         default: return "INSTR_UNKNOWN";
10016         }
10017 }
10018
10019 typedef void
10020 (*instruction_export_t)(struct instruction *, FILE *);
10021
10022 static void
10023 instr_io_export(struct instruction *instr, FILE *f)
10024 {
10025         uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10026
10027         /* n_io, n_io_imm, n_hdrs. */
10028         if (instr->type == INSTR_RX ||
10029             instr->type == INSTR_TX ||
10030             instr->type == INSTR_HDR_EXTRACT_M ||
10031             (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10032                 n_io = 1;
10033
10034         if (instr->type == INSTR_TX_I)
10035                 n_io_imm = 1;
10036
10037         if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10038                 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10039
10040         if (instr->type == INSTR_HDR_EXTRACT_M ||
10041             instr->type == INSTR_HDR_LOOKAHEAD ||
10042             instr->type == INSTR_HDR_EMIT)
10043                 n_hdrs = 1;
10044
10045         if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10046                 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10047
10048         /* instr. */
10049         fprintf(f,
10050                 "\t{\n"
10051                 "\t\t.type = %s,\n",
10052                 instr_type_to_name(instr));
10053
10054         /* instr.io. */
10055         fprintf(f,
10056                 "\t\t.io = {\n");
10057
10058         /* instr.io.io. */
10059         if (n_io)
10060                 fprintf(f,
10061                         "\t\t\t.io = {\n"
10062                         "\t\t\t\t.offset = %u,\n"
10063                         "\t\t\t\t.n_bits = %u,\n"
10064                         "\t\t\t},\n",
10065                         instr->io.io.offset,
10066                         instr->io.io.n_bits);
10067
10068         if (n_io_imm)
10069                 fprintf(f,
10070                         "\t\t\t.io = {\n"
10071                         "\t\t\t\t.val = %u,\n"
10072                         "\t\t\t},\n",
10073                         instr->io.io.val);
10074
10075         /* instr.io.hdr. */
10076         if (n_hdrs) {
10077                 fprintf(f,
10078                         "\t\t.hdr = {\n");
10079
10080                 /* instr.io.hdr.header_id. */
10081                 fprintf(f,
10082                         "\t\t\t.header_id = {");
10083
10084                 for (i = 0; i < n_hdrs; i++)
10085                         fprintf(f,
10086                                 "%u, ",
10087                                 instr->io.hdr.header_id[i]);
10088
10089                 fprintf(f,
10090                         "},\n");
10091
10092                 /* instr.io.hdr.struct_id. */
10093                 fprintf(f,
10094                         "\t\t\t.struct_id = {");
10095
10096                 for (i = 0; i < n_hdrs; i++)
10097                         fprintf(f,
10098                                 "%u, ",
10099                                 instr->io.hdr.struct_id[i]);
10100
10101                 fprintf(f,
10102                         "},\n");
10103
10104                 /* instr.io.hdr.n_bytes. */
10105                 fprintf(f,
10106                         "\t\t\t.n_bytes = {");
10107
10108                 for (i = 0; i < n_hdrs; i++)
10109                         fprintf(f,
10110                                 "%u, ",
10111                                 instr->io.hdr.n_bytes[i]);
10112
10113                 fprintf(f,
10114                         "},\n");
10115
10116                 /* instr.io.hdr - closing curly brace. */
10117                 fprintf(f,
10118                         "\t\t\t}\n,");
10119         }
10120
10121         /* instr.io - closing curly brace. */
10122         fprintf(f,
10123                 "\t\t},\n");
10124
10125         /* instr - closing curly brace. */
10126         fprintf(f,
10127                 "\t},\n");
10128 }
10129
10130 static void
10131 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10132 {
10133         fprintf(f,
10134                 "\t{\n"
10135                 "\t\t.type = %s,\n"
10136                 "\t\t.valid = {\n"
10137                 "\t\t\t.header_id = %u,\n"
10138                 "\t\t},\n"
10139                 "\t},\n",
10140                 instr_type_to_name(instr),
10141                 instr->valid.header_id);
10142 }
10143
10144 static void
10145 instr_mov_export(struct instruction *instr, FILE *f)
10146 {
10147         if (instr->type != INSTR_MOV_I)
10148                 fprintf(f,
10149                         "\t{\n"
10150                         "\t\t.type = %s,\n"
10151                         "\t\t.mov = {\n"
10152                         "\t\t\t.dst = {\n"
10153                         "\t\t\t\t.struct_id = %u,\n"
10154                         "\t\t\t\t.n_bits = %u,\n"
10155                         "\t\t\t\t.offset = %u,\n"
10156                         "\t\t\t},\n"
10157                         "\t\t\t.src = {\n"
10158                         "\t\t\t\t.struct_id = %u,\n"
10159                         "\t\t\t\t.n_bits = %u,\n"
10160                         "\t\t\t\t.offset = %u,\n"
10161                         "\t\t\t},\n"
10162                         "\t\t},\n"
10163                         "\t},\n",
10164                         instr_type_to_name(instr),
10165                         instr->mov.dst.struct_id,
10166                         instr->mov.dst.n_bits,
10167                         instr->mov.dst.offset,
10168                         instr->mov.src.struct_id,
10169                         instr->mov.src.n_bits,
10170                         instr->mov.src.offset);
10171         else
10172                 fprintf(f,
10173                         "\t{\n"
10174                         "\t\t.type = %s,\n"
10175                         "\t\t.mov = {\n"
10176                         "\t\t\t.dst = {\n"
10177                         "\t\t\t\t.struct_id = %u,\n"
10178                         "\t\t\t\t.n_bits = %u,\n"
10179                         "\t\t\t\t.offset = %u,\n"
10180                         "\t\t\t}\n,"
10181                         "\t\t\t.src_val = %" PRIu64 ",\n"
10182                         "\t\t},\n"
10183                         "\t},\n",
10184                         instr_type_to_name(instr),
10185                         instr->mov.dst.struct_id,
10186                         instr->mov.dst.n_bits,
10187                         instr->mov.dst.offset,
10188                         instr->mov.src_val);
10189 }
10190
10191 static void
10192 instr_dma_ht_export(struct instruction *instr, FILE *f)
10193 {
10194         uint32_t n_dma = 0, i;
10195
10196         /* n_dma. */
10197         n_dma = 1 + (instr->type - INSTR_DMA_HT);
10198
10199         /* instr. */
10200         fprintf(f,
10201                 "\t{\n"
10202                 "\t\t.type = %s,\n",
10203                 instr_type_to_name(instr));
10204
10205         /* instr.dma. */
10206         fprintf(f,
10207                 "\t\t.dma = {\n");
10208
10209         /* instr.dma.dst. */
10210         fprintf(f,
10211                 "\t\t\t.dst = {\n");
10212
10213         /* instr.dma.dst.header_id. */
10214         fprintf(f,
10215                 "\t\t\t\t.header_id = {");
10216
10217         for (i = 0; i < n_dma; i++)
10218                 fprintf(f,
10219                         "%u, ",
10220                         instr->dma.dst.header_id[i]);
10221
10222         fprintf(f,
10223                 "},\n");
10224
10225         /* instr.dma.dst.struct_id. */
10226         fprintf(f,
10227                 "\t\t\t\t.struct_id = {");
10228
10229         for (i = 0; i < n_dma; i++)
10230                 fprintf(f,
10231                         "%u, ",
10232                         instr->dma.dst.struct_id[i]);
10233
10234         fprintf(f,
10235                 "},\n");
10236
10237         /* instr.dma.dst - closing curly brace. */
10238         fprintf(f,
10239                 "\t\t\t},\n");
10240
10241         /* instr.dma.src. */
10242         fprintf(f,
10243                 "\t\t\t.src = {\n");
10244
10245         /* instr.dma.src.offset. */
10246         fprintf(f,
10247                 "\t\t\t\t.offset = {");
10248
10249         for (i = 0; i < n_dma; i++)
10250                 fprintf(f,
10251                         "%u, ",
10252                         instr->dma.src.offset[i]);
10253
10254         fprintf(f,
10255                 "},\n");
10256
10257         /* instr.dma.src - closing curly brace. */
10258         fprintf(f,
10259                 "\t\t\t},\n");
10260
10261         /* instr.dma.n_bytes. */
10262         fprintf(f,
10263                 "\t\t\t.n_bytes = {");
10264
10265         for (i = 0; i < n_dma; i++)
10266                 fprintf(f,
10267                         "%u, ",
10268                         instr->dma.n_bytes[i]);
10269
10270         fprintf(f,
10271                 "},\n");
10272
10273         /* instr.dma - closing curly brace. */
10274         fprintf(f,
10275                 "\t\t},\n");
10276
10277         /* instr - closing curly brace. */
10278         fprintf(f,
10279                 "\t},\n");
10280 }
10281
10282 static void
10283 instr_alu_export(struct instruction *instr, FILE *f)
10284 {
10285         int imm = 0;
10286
10287         if (instr->type == INSTR_ALU_ADD_MI ||
10288             instr->type == INSTR_ALU_ADD_HI ||
10289             instr->type == INSTR_ALU_SUB_MI ||
10290             instr->type == INSTR_ALU_SUB_HI ||
10291             instr->type == INSTR_ALU_SHL_MI ||
10292             instr->type == INSTR_ALU_SHL_HI ||
10293             instr->type == INSTR_ALU_SHR_MI ||
10294             instr->type == INSTR_ALU_SHR_HI ||
10295             instr->type == INSTR_ALU_AND_I ||
10296             instr->type == INSTR_ALU_OR_I ||
10297             instr->type == INSTR_ALU_XOR_I)
10298                 imm = 1;
10299
10300         if (!imm)
10301                 fprintf(f,
10302                         "\t{\n"
10303                         "\t\t.type = %s,\n"
10304                         "\t\t.alu = {\n"
10305                         "\t\t\t.dst = {\n"
10306                         "\t\t\t\t.struct_id = %u,\n"
10307                         "\t\t\t\t.n_bits = %u,\n"
10308                         "\t\t\t\t.offset = %u,\n"
10309                         "\t\t\t},\n"
10310                         "\t\t\t.src = {\n"
10311                         "\t\t\t\t.struct_id = %u,\n"
10312                         "\t\t\t\t.n_bits = %u,\n"
10313                         "\t\t\t\t.offset = %u,\n"
10314                         "\t\t\t},\n"
10315                         "\t\t},\n"
10316                         "\t},\n",
10317                         instr_type_to_name(instr),
10318                         instr->alu.dst.struct_id,
10319                         instr->alu.dst.n_bits,
10320                         instr->alu.dst.offset,
10321                         instr->alu.src.struct_id,
10322                         instr->alu.src.n_bits,
10323                         instr->alu.src.offset);
10324         else
10325                 fprintf(f,
10326                         "\t{\n"
10327                         "\t\t.type = %s,\n"
10328                         "\t\t.alu = {\n"
10329                         "\t\t\t.dst = {\n"
10330                         "\t\t\t\t.struct_id = %u,\n"
10331                         "\t\t\t\t.n_bits = %u,\n"
10332                         "\t\t\t\t.offset = %u,\n"
10333                         "\t\t\t}\n,"
10334                         "\t\t\t.src_val = %" PRIu64 ",\n"
10335                         "\t\t},\n"
10336                         "\t},\n",
10337                         instr_type_to_name(instr),
10338                         instr->alu.dst.struct_id,
10339                         instr->alu.dst.n_bits,
10340                         instr->alu.dst.offset,
10341                         instr->alu.src_val);
10342 }
10343
10344 static void
10345 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10346 {
10347         int prefetch  = 0, idx_imm = 0, src_imm = 0;
10348
10349         if (instr->type == INSTR_REGPREFETCH_RH ||
10350             instr->type == INSTR_REGPREFETCH_RM ||
10351             instr->type == INSTR_REGPREFETCH_RI)
10352                 prefetch = 1;
10353
10354         /* index is the 3rd operand for the regrd instruction and the 2nd
10355          * operand for the regwr and regadd instructions.
10356          */
10357         if (instr->type == INSTR_REGPREFETCH_RI ||
10358             instr->type == INSTR_REGRD_HRI ||
10359             instr->type == INSTR_REGRD_MRI ||
10360             instr->type == INSTR_REGWR_RIH ||
10361             instr->type == INSTR_REGWR_RIM ||
10362             instr->type == INSTR_REGWR_RII ||
10363             instr->type == INSTR_REGADD_RIH ||
10364             instr->type == INSTR_REGADD_RIM ||
10365             instr->type == INSTR_REGADD_RII)
10366                 idx_imm = 1;
10367
10368         /* src is the 3rd operand for the regwr and regadd instructions. */
10369         if (instr->type == INSTR_REGWR_RHI ||
10370             instr->type == INSTR_REGWR_RMI ||
10371             instr->type == INSTR_REGWR_RII ||
10372             instr->type == INSTR_REGADD_RHI ||
10373             instr->type == INSTR_REGADD_RMI ||
10374             instr->type == INSTR_REGADD_RII)
10375                 src_imm = 1;
10376
10377         /* instr.regarray.regarray_id. */
10378         fprintf(f,
10379                 "\t{\n"
10380                 "\t\t.type = %s,\n"
10381                 "\t\t.regarray = {\n"
10382                 "\t\t\t.regarray_id = %u,\n",
10383                 instr_type_to_name(instr),
10384                 instr->regarray.regarray_id);
10385
10386         /* instr.regarray.idx / instr.regarray.idx_val. */
10387         if (!idx_imm)
10388                 fprintf(f,
10389                         "\t\t\t\t.idx = {\n"
10390                         "\t\t\t\t\t.struct_id = %u,\n"
10391                         "\t\t\t\t\t.n_bits = %u,\n"
10392                         "\t\t\t\t\t.offset = %u,\n"
10393                         "\t\t\t\t},\n",
10394                         instr->regarray.idx.struct_id,
10395                         instr->regarray.idx.n_bits,
10396                         instr->regarray.idx.offset);
10397         else
10398                 fprintf(f,
10399                         "\t\t\t\t.idx_val = %u,\n",
10400                         instr->regarray.idx_val);
10401
10402         /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10403         if (!prefetch) {
10404                 if (!src_imm)
10405                         fprintf(f,
10406                                 "\t\t\t\t.dstsrc = {\n"
10407                                 "\t\t\t\t\t.struct_id = %u,\n"
10408                                 "\t\t\t\t\t.n_bits = %u,\n"
10409                                 "\t\t\t\t\t.offset = %u,\n"
10410                                 "\t\t\t\t},\n",
10411                                 instr->regarray.dstsrc.struct_id,
10412                                 instr->regarray.dstsrc.n_bits,
10413                                 instr->regarray.dstsrc.offset);
10414                 else
10415                         fprintf(f,
10416                                 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10417                                 instr->regarray.dstsrc_val);
10418         }
10419
10420         /* instr.regarray and instr - closing curly braces. */
10421         fprintf(f,
10422                 "\t\t},\n"
10423                 "\t},\n");
10424 }
10425
10426 static void
10427 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10428 {
10429         int prefetch  = 0, idx_imm = 0, color_in_imm = 0;
10430
10431         if (instr->type == INSTR_METPREFETCH_H ||
10432             instr->type == INSTR_METPREFETCH_M ||
10433             instr->type == INSTR_METPREFETCH_I)
10434                 prefetch = 1;
10435
10436         /* idx_imm. */
10437         if (instr->type == INSTR_METPREFETCH_I ||
10438             instr->type == INSTR_METER_IHM ||
10439             instr->type == INSTR_METER_IHI ||
10440             instr->type == INSTR_METER_IMM ||
10441             instr->type == INSTR_METER_IMI)
10442                 idx_imm = 1;
10443
10444         /* color_in_imm. */
10445         if (instr->type == INSTR_METER_HHI ||
10446             instr->type == INSTR_METER_HMI ||
10447             instr->type == INSTR_METER_MHI ||
10448             instr->type == INSTR_METER_MMI ||
10449             instr->type == INSTR_METER_IHI ||
10450             instr->type == INSTR_METER_IMI)
10451                 color_in_imm = 1;
10452
10453         /* instr.meter.metarray_id. */
10454         fprintf(f,
10455                 "\t{\n"
10456                 "\t\t.type = %s,\n"
10457                 "\t\t.meter = {\n"
10458                 "\t\t\t.metarray_id = %u,\n",
10459                 instr_type_to_name(instr),
10460                 instr->meter.metarray_id);
10461
10462         /* instr.meter.idx / instr.meter.idx_val. */
10463         if (!idx_imm)
10464                 fprintf(f,
10465                         "\t\t\t.idx = {\n"
10466                         "\t\t\t\t.struct_id = %u,\n"
10467                         "\t\t\t\t.n_bits = %u,\n"
10468                         "\t\t\t\t.offset = %u,\n"
10469                         "\t\t\t},\n",
10470                         instr->meter.idx.struct_id,
10471                         instr->meter.idx.n_bits,
10472                         instr->meter.idx.offset);
10473         else
10474                 fprintf(f,
10475                         "\t\t\t.idx_val = %u,\n",
10476                         instr->meter.idx_val);
10477
10478         if (!prefetch) {
10479                 /* instr.meter.length. */
10480                 fprintf(f,
10481                         "\t\t\t.length = {\n"
10482                         "\t\t\t\t.struct_id = %u,\n"
10483                         "\t\t\t\t.n_bits = %u,\n"
10484                         "\t\t\t\t.offset = %u,\n"
10485                         "\t\t\t},\n",
10486                         instr->meter.length.struct_id,
10487                         instr->meter.length.n_bits,
10488                         instr->meter.length.offset);
10489
10490                 /* instr.meter.color_in / instr.meter.color_in_val. */
10491                 if (!color_in_imm)
10492                         fprintf(f,
10493                                 "\t\t\t.color_in = {\n"
10494                                 "\t\t\t\t.struct_id = %u,\n"
10495                                 "\t\t\t\t.n_bits = %u,\n"
10496                                 "\t\t\t\t.offset = %u,\n"
10497                                 "\t\t\t},\n",
10498                                 instr->meter.color_in.struct_id,
10499                                 instr->meter.color_in.n_bits,
10500                                 instr->meter.color_in.offset);
10501                 else
10502                         fprintf(f,
10503                                 "\t\t\t.color_in_val = %u,\n",
10504                                 (uint32_t)instr->meter.color_in_val);
10505
10506                 /* instr.meter.color_out. */
10507                 fprintf(f,
10508                         "\t\t\t.color_out = {\n"
10509                         "\t\t\t\t.struct_id = %u,\n"
10510                         "\t\t\t\t.n_bits = %u,\n"
10511                         "\t\t\t\t.offset = %u,\n"
10512                         "\t\t\t},\n",
10513                         instr->meter.color_out.struct_id,
10514                         instr->meter.color_out.n_bits,
10515                         instr->meter.color_out.offset);
10516         }
10517
10518         /* instr.meter and instr - closing curly braces. */
10519         fprintf(f,
10520                 "\t\t},\n"
10521                 "\t},\n");
10522 }
10523
10524 static void
10525 instr_table_export(struct instruction *instr,
10526                 FILE *f)
10527 {
10528         fprintf(f,
10529                 "\t{\n"
10530                 "\t\t.type = %s,\n"
10531                 "\t\t.table = {\n"
10532                 "\t\t\t.table_id = %u,\n"
10533                 "\t\t},\n"
10534                 "\t},\n",
10535                 instr_type_to_name(instr),
10536                 instr->table.table_id);
10537 }
10538
10539 static void
10540 instr_learn_export(struct instruction *instr, FILE *f)
10541 {
10542         fprintf(f,
10543                 "\t{\n"
10544                 "\t\t.type = %s,\n"
10545                 "\t\t.learn = {\n"
10546                 "\t\t\t\t.action_id = %u,\n"
10547                 "\t\t},\n"
10548                 "\t},\n",
10549                 instr_type_to_name(instr),
10550                 instr->learn.action_id);
10551 }
10552
10553 static void
10554 instr_forget_export(struct instruction *instr, FILE *f)
10555 {
10556         fprintf(f,
10557                 "\t{\n"
10558                 "\t\t.type = %s,\n"
10559                 "\t},\n",
10560                 instr_type_to_name(instr));
10561 }
10562
10563 static void
10564 instr_extern_export(struct instruction *instr, FILE *f)
10565 {
10566         if (instr->type == INSTR_EXTERN_OBJ)
10567                 fprintf(f,
10568                         "\t{\n"
10569                         "\t\t.type = %s,\n"
10570                         "\t\t.ext_obj = {\n"
10571                         "\t\t\t.ext_obj_id = %u,\n"
10572                         "\t\t\t.func_id = %u,\n"
10573                         "\t\t},\n"
10574                         "\t},\n",
10575                         instr_type_to_name(instr),
10576                         instr->ext_obj.ext_obj_id,
10577                         instr->ext_obj.func_id);
10578         else
10579                 fprintf(f,
10580                         "\t{\n"
10581                         "\t\t.type = %s,\n"
10582                         "\t\t.ext_func = {\n"
10583                         "\t\t\t.ext_func_id = %u,\n"
10584                         "\t\t},\n"
10585                         "\t},\n",
10586                         instr_type_to_name(instr),
10587                         instr->ext_func.ext_func_id);
10588 }
10589
10590 static void
10591 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10592 {
10593         fprintf(f,
10594                 "\t{\n"
10595                 "\t\t.type = %s,\n"
10596                 "\t\t.jmp = {\n"
10597                 "\t\t\t.ip = NULL,\n",
10598                 instr_type_to_name(instr));
10599
10600         switch (instr->type) {
10601         case INSTR_JMP_VALID:
10602         case INSTR_JMP_INVALID:
10603                 fprintf(f,
10604                         "\t\t\t.header_id = %u,\n",
10605                         instr->jmp.header_id);
10606                 break;
10607
10608         case INSTR_JMP_ACTION_HIT:
10609         case INSTR_JMP_ACTION_MISS:
10610                 fprintf(f,
10611                         "\t\t\t.action_id = %u,\n",
10612                         instr->jmp.action_id);
10613                 break;
10614
10615         case INSTR_JMP_EQ:
10616         case INSTR_JMP_EQ_MH:
10617         case INSTR_JMP_EQ_HM:
10618         case INSTR_JMP_EQ_HH:
10619         case INSTR_JMP_NEQ:
10620         case INSTR_JMP_NEQ_MH:
10621         case INSTR_JMP_NEQ_HM:
10622         case INSTR_JMP_NEQ_HH:
10623         case INSTR_JMP_LT:
10624         case INSTR_JMP_LT_MH:
10625         case INSTR_JMP_LT_HM:
10626         case INSTR_JMP_LT_HH:
10627         case INSTR_JMP_GT:
10628         case INSTR_JMP_GT_MH:
10629         case INSTR_JMP_GT_HM:
10630         case INSTR_JMP_GT_HH:
10631                 fprintf(f,
10632                         "\t\t\t.a = {\n"
10633                         "\t\t\t\t.struct_id = %u,\n"
10634                         "\t\t\t\t.n_bits = %u,\n"
10635                         "\t\t\t\t.offset = %u,\n"
10636                         "\t\t\t},\n"
10637                         "\t\t\t.b = {\n"
10638                         "\t\t\t\t.struct_id = %u,\n"
10639                         "\t\t\t\t.n_bits = %u,\n"
10640                         "\t\t\t\t.offset = %u,\n"
10641                         "\t\t\t},\n",
10642                         instr->jmp.a.struct_id,
10643                         instr->jmp.a.n_bits,
10644                         instr->jmp.a.offset,
10645                         instr->jmp.b.struct_id,
10646                         instr->jmp.b.n_bits,
10647                         instr->jmp.b.offset);
10648                 break;
10649
10650         case INSTR_JMP_EQ_I:
10651         case INSTR_JMP_NEQ_I:
10652         case INSTR_JMP_LT_MI:
10653         case INSTR_JMP_LT_HI:
10654         case INSTR_JMP_GT_MI:
10655         case INSTR_JMP_GT_HI:
10656                 fprintf(f,
10657                         "\t\t\t.a = {\n"
10658                         "\t\t\t\t.struct_id = %u,\n"
10659                         "\t\t\t\t.n_bits = %u,\n"
10660                         "\t\t\t\t.offset = %u,\n"
10661                         "\t\t\t}\n,"
10662                         "\t\t\t.b_val = %" PRIu64 ",\n",
10663                         instr->jmp.a.struct_id,
10664                         instr->jmp.a.n_bits,
10665                         instr->jmp.a.offset,
10666                         instr->jmp.b_val);
10667                 break;
10668
10669         default:
10670                 break;
10671         }
10672
10673         fprintf(f,
10674                 "\t\t},\n"
10675                 "\t},\n");
10676 }
10677
10678 static void
10679 instr_return_export(struct instruction *instr,
10680                 FILE *f)
10681 {
10682         fprintf(f,
10683                 "\t{\n"
10684                 "\t\t.type = %s,\n",
10685                 instr_type_to_name(instr));
10686
10687         fprintf(f,
10688                 "\t},\n");
10689 }
10690
10691 static instruction_export_t export_table[] = {
10692         [INSTR_RX] = instr_io_export,
10693
10694         [INSTR_TX] = instr_io_export,
10695         [INSTR_TX_I] = instr_io_export,
10696
10697         [INSTR_HDR_EXTRACT] = instr_io_export,
10698         [INSTR_HDR_EXTRACT2] = instr_io_export,
10699         [INSTR_HDR_EXTRACT3] = instr_io_export,
10700         [INSTR_HDR_EXTRACT4] = instr_io_export,
10701         [INSTR_HDR_EXTRACT5] = instr_io_export,
10702         [INSTR_HDR_EXTRACT6] = instr_io_export,
10703         [INSTR_HDR_EXTRACT7] = instr_io_export,
10704         [INSTR_HDR_EXTRACT8] = instr_io_export,
10705
10706         [INSTR_HDR_EXTRACT_M] = instr_io_export,
10707
10708         [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10709
10710         [INSTR_HDR_EMIT] = instr_io_export,
10711         [INSTR_HDR_EMIT_TX] = instr_io_export,
10712         [INSTR_HDR_EMIT2_TX] = instr_io_export,
10713         [INSTR_HDR_EMIT3_TX] = instr_io_export,
10714         [INSTR_HDR_EMIT4_TX] = instr_io_export,
10715         [INSTR_HDR_EMIT5_TX] = instr_io_export,
10716         [INSTR_HDR_EMIT6_TX] = instr_io_export,
10717         [INSTR_HDR_EMIT7_TX] = instr_io_export,
10718         [INSTR_HDR_EMIT8_TX] = instr_io_export,
10719
10720         [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10721         [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10722
10723         [INSTR_MOV] = instr_mov_export,
10724         [INSTR_MOV_MH] = instr_mov_export,
10725         [INSTR_MOV_HM] = instr_mov_export,
10726         [INSTR_MOV_HH] = instr_mov_export,
10727         [INSTR_MOV_I] = instr_mov_export,
10728
10729         [INSTR_DMA_HT]  = instr_dma_ht_export,
10730         [INSTR_DMA_HT2] = instr_dma_ht_export,
10731         [INSTR_DMA_HT3] = instr_dma_ht_export,
10732         [INSTR_DMA_HT4] = instr_dma_ht_export,
10733         [INSTR_DMA_HT5] = instr_dma_ht_export,
10734         [INSTR_DMA_HT6] = instr_dma_ht_export,
10735         [INSTR_DMA_HT7] = instr_dma_ht_export,
10736         [INSTR_DMA_HT8] = instr_dma_ht_export,
10737
10738         [INSTR_ALU_ADD] = instr_alu_export,
10739         [INSTR_ALU_ADD_MH] = instr_alu_export,
10740         [INSTR_ALU_ADD_HM] = instr_alu_export,
10741         [INSTR_ALU_ADD_HH] = instr_alu_export,
10742         [INSTR_ALU_ADD_MI] = instr_alu_export,
10743         [INSTR_ALU_ADD_HI] = instr_alu_export,
10744
10745         [INSTR_ALU_SUB] = instr_alu_export,
10746         [INSTR_ALU_SUB_MH] = instr_alu_export,
10747         [INSTR_ALU_SUB_HM] = instr_alu_export,
10748         [INSTR_ALU_SUB_HH] = instr_alu_export,
10749         [INSTR_ALU_SUB_MI] = instr_alu_export,
10750         [INSTR_ALU_SUB_HI] = instr_alu_export,
10751
10752         [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10753         [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10754         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10755         [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10756
10757         [INSTR_ALU_AND] = instr_alu_export,
10758         [INSTR_ALU_AND_MH] = instr_alu_export,
10759         [INSTR_ALU_AND_HM] = instr_alu_export,
10760         [INSTR_ALU_AND_HH] = instr_alu_export,
10761         [INSTR_ALU_AND_I] = instr_alu_export,
10762
10763         [INSTR_ALU_OR] = instr_alu_export,
10764         [INSTR_ALU_OR_MH] = instr_alu_export,
10765         [INSTR_ALU_OR_HM] = instr_alu_export,
10766         [INSTR_ALU_OR_HH] = instr_alu_export,
10767         [INSTR_ALU_OR_I] = instr_alu_export,
10768
10769         [INSTR_ALU_XOR] = instr_alu_export,
10770         [INSTR_ALU_XOR_MH] = instr_alu_export,
10771         [INSTR_ALU_XOR_HM] = instr_alu_export,
10772         [INSTR_ALU_XOR_HH] = instr_alu_export,
10773         [INSTR_ALU_XOR_I] = instr_alu_export,
10774
10775         [INSTR_ALU_SHL] = instr_alu_export,
10776         [INSTR_ALU_SHL_MH] = instr_alu_export,
10777         [INSTR_ALU_SHL_HM] = instr_alu_export,
10778         [INSTR_ALU_SHL_HH] = instr_alu_export,
10779         [INSTR_ALU_SHL_MI] = instr_alu_export,
10780         [INSTR_ALU_SHL_HI] = instr_alu_export,
10781
10782         [INSTR_ALU_SHR] = instr_alu_export,
10783         [INSTR_ALU_SHR_MH] = instr_alu_export,
10784         [INSTR_ALU_SHR_HM] = instr_alu_export,
10785         [INSTR_ALU_SHR_HH] = instr_alu_export,
10786         [INSTR_ALU_SHR_MI] = instr_alu_export,
10787         [INSTR_ALU_SHR_HI] = instr_alu_export,
10788
10789         [INSTR_REGPREFETCH_RH] = instr_reg_export,
10790         [INSTR_REGPREFETCH_RM] = instr_reg_export,
10791         [INSTR_REGPREFETCH_RI] = instr_reg_export,
10792
10793         [INSTR_REGRD_HRH] = instr_reg_export,
10794         [INSTR_REGRD_HRM] = instr_reg_export,
10795         [INSTR_REGRD_MRH] = instr_reg_export,
10796         [INSTR_REGRD_MRM] = instr_reg_export,
10797         [INSTR_REGRD_HRI] = instr_reg_export,
10798         [INSTR_REGRD_MRI] = instr_reg_export,
10799
10800         [INSTR_REGWR_RHH] = instr_reg_export,
10801         [INSTR_REGWR_RHM] = instr_reg_export,
10802         [INSTR_REGWR_RMH] = instr_reg_export,
10803         [INSTR_REGWR_RMM] = instr_reg_export,
10804         [INSTR_REGWR_RHI] = instr_reg_export,
10805         [INSTR_REGWR_RMI] = instr_reg_export,
10806         [INSTR_REGWR_RIH] = instr_reg_export,
10807         [INSTR_REGWR_RIM] = instr_reg_export,
10808         [INSTR_REGWR_RII] = instr_reg_export,
10809
10810         [INSTR_REGADD_RHH] = instr_reg_export,
10811         [INSTR_REGADD_RHM] = instr_reg_export,
10812         [INSTR_REGADD_RMH] = instr_reg_export,
10813         [INSTR_REGADD_RMM] = instr_reg_export,
10814         [INSTR_REGADD_RHI] = instr_reg_export,
10815         [INSTR_REGADD_RMI] = instr_reg_export,
10816         [INSTR_REGADD_RIH] = instr_reg_export,
10817         [INSTR_REGADD_RIM] = instr_reg_export,
10818         [INSTR_REGADD_RII] = instr_reg_export,
10819
10820         [INSTR_METPREFETCH_H] = instr_meter_export,
10821         [INSTR_METPREFETCH_M] = instr_meter_export,
10822         [INSTR_METPREFETCH_I] = instr_meter_export,
10823
10824         [INSTR_METER_HHM] = instr_meter_export,
10825         [INSTR_METER_HHI] = instr_meter_export,
10826         [INSTR_METER_HMM] = instr_meter_export,
10827         [INSTR_METER_HMI] = instr_meter_export,
10828         [INSTR_METER_MHM] = instr_meter_export,
10829         [INSTR_METER_MHI] = instr_meter_export,
10830         [INSTR_METER_MMM] = instr_meter_export,
10831         [INSTR_METER_MMI] = instr_meter_export,
10832         [INSTR_METER_IHM] = instr_meter_export,
10833         [INSTR_METER_IHI] = instr_meter_export,
10834         [INSTR_METER_IMM] = instr_meter_export,
10835         [INSTR_METER_IMI] = instr_meter_export,
10836
10837         [INSTR_TABLE] = instr_table_export,
10838         [INSTR_TABLE_AF] = instr_table_export,
10839         [INSTR_SELECTOR] = instr_table_export,
10840         [INSTR_LEARNER] = instr_table_export,
10841         [INSTR_LEARNER_AF] = instr_table_export,
10842
10843         [INSTR_LEARNER_LEARN] = instr_learn_export,
10844         [INSTR_LEARNER_FORGET] = instr_forget_export,
10845
10846         [INSTR_EXTERN_OBJ] = instr_extern_export,
10847         [INSTR_EXTERN_FUNC] = instr_extern_export,
10848
10849         [INSTR_JMP] = instr_jmp_export,
10850         [INSTR_JMP_VALID] = instr_jmp_export,
10851         [INSTR_JMP_INVALID] = instr_jmp_export,
10852         [INSTR_JMP_HIT] = instr_jmp_export,
10853         [INSTR_JMP_MISS] = instr_jmp_export,
10854         [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
10855         [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
10856
10857         [INSTR_JMP_EQ] = instr_jmp_export,
10858         [INSTR_JMP_EQ_MH] = instr_jmp_export,
10859         [INSTR_JMP_EQ_HM] = instr_jmp_export,
10860         [INSTR_JMP_EQ_HH] = instr_jmp_export,
10861         [INSTR_JMP_EQ_I] = instr_jmp_export,
10862
10863         [INSTR_JMP_NEQ] = instr_jmp_export,
10864         [INSTR_JMP_NEQ_MH] = instr_jmp_export,
10865         [INSTR_JMP_NEQ_HM] = instr_jmp_export,
10866         [INSTR_JMP_NEQ_HH] = instr_jmp_export,
10867         [INSTR_JMP_NEQ_I] = instr_jmp_export,
10868
10869         [INSTR_JMP_LT] = instr_jmp_export,
10870         [INSTR_JMP_LT_MH] = instr_jmp_export,
10871         [INSTR_JMP_LT_HM] = instr_jmp_export,
10872         [INSTR_JMP_LT_HH] = instr_jmp_export,
10873         [INSTR_JMP_LT_MI] = instr_jmp_export,
10874         [INSTR_JMP_LT_HI] = instr_jmp_export,
10875
10876         [INSTR_JMP_GT] = instr_jmp_export,
10877         [INSTR_JMP_GT_MH] = instr_jmp_export,
10878         [INSTR_JMP_GT_HM] = instr_jmp_export,
10879         [INSTR_JMP_GT_HH] = instr_jmp_export,
10880         [INSTR_JMP_GT_MI] = instr_jmp_export,
10881         [INSTR_JMP_GT_HI] = instr_jmp_export,
10882
10883         [INSTR_RETURN] = instr_return_export,
10884 };
10885
10886 static void
10887 action_data_codegen(struct action *a, FILE *f)
10888 {
10889         uint32_t i;
10890
10891         fprintf(f,
10892                 "static const struct instruction action_%s_instructions[] = {\n",
10893                 a->name);
10894
10895         for (i = 0; i < a->n_instructions; i++) {
10896                 struct instruction *instr = &a->instructions[i];
10897                 instruction_export_t func = export_table[instr->type];
10898
10899                 func(instr, f);
10900         }
10901
10902         fprintf(f, "};\n");
10903 }
10904
10905 static const char *
10906 instr_type_to_func(struct instruction *instr)
10907 {
10908         switch (instr->type) {
10909         case INSTR_RX: return NULL;
10910
10911         case INSTR_TX: return "__instr_tx_exec";
10912         case INSTR_TX_I: return "__instr_tx_i_exec";
10913
10914         case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
10915         case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
10916         case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
10917         case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
10918         case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
10919         case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
10920         case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
10921         case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
10922
10923         case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
10924
10925         case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
10926
10927         case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
10928         case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
10929         case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
10930         case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
10931         case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
10932         case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
10933         case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
10934         case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
10935         case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
10936
10937         case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
10938         case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
10939
10940         case INSTR_MOV: return "__instr_mov_exec";
10941         case INSTR_MOV_MH: return "__instr_mov_mh_exec";
10942         case INSTR_MOV_HM: return "__instr_mov_hm_exec";
10943         case INSTR_MOV_HH: return "__instr_mov_hh_exec";
10944         case INSTR_MOV_I: return "__instr_mov_i_exec";
10945
10946         case INSTR_DMA_HT: return "__instr_dma_ht_exec";
10947         case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
10948         case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
10949         case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
10950         case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
10951         case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
10952         case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
10953         case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
10954
10955         case INSTR_ALU_ADD: return "__instr_alu_add_exec";
10956         case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
10957         case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
10958         case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
10959         case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
10960         case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
10961
10962         case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
10963         case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
10964         case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
10965         case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
10966         case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
10967         case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
10968
10969         case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
10970         case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
10971         case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
10972         case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
10973
10974         case INSTR_ALU_AND: return "__instr_alu_and_exec";
10975         case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
10976         case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
10977         case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
10978         case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
10979
10980         case INSTR_ALU_OR: return "__instr_alu_or_exec";
10981         case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
10982         case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
10983         case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
10984         case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
10985
10986         case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
10987         case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
10988         case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
10989         case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
10990         case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
10991
10992         case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
10993         case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
10994         case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
10995         case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
10996         case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
10997         case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
10998
10999         case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11000         case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11001         case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11002         case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11003         case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11004         case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11005
11006         case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11007         case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11008         case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11009
11010         case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11011         case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11012         case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11013         case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11014         case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11015         case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11016
11017         case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11018         case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11019         case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11020         case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11021         case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11022         case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11023         case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11024         case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11025         case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11026
11027         case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11028         case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11029         case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11030         case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11031         case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11032         case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11033         case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11034         case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11035         case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11036
11037         case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11038         case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11039         case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11040
11041         case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11042         case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11043         case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11044         case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11045         case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11046         case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11047         case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11048         case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11049         case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11050         case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11051         case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11052         case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11053
11054         case INSTR_TABLE: return NULL;
11055         case INSTR_TABLE_AF: return NULL;
11056         case INSTR_SELECTOR: return NULL;
11057         case INSTR_LEARNER: return NULL;
11058         case INSTR_LEARNER_AF: return NULL;
11059
11060         case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11061         case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11062
11063         case INSTR_EXTERN_OBJ: return NULL;
11064         case INSTR_EXTERN_FUNC: return NULL;
11065
11066         case INSTR_JMP: return NULL;
11067         case INSTR_JMP_VALID: return NULL;
11068         case INSTR_JMP_INVALID: return NULL;
11069         case INSTR_JMP_HIT: return NULL;
11070         case INSTR_JMP_MISS: return NULL;
11071         case INSTR_JMP_ACTION_HIT: return NULL;
11072         case INSTR_JMP_ACTION_MISS: return NULL;
11073         case INSTR_JMP_EQ: return NULL;
11074         case INSTR_JMP_EQ_MH: return NULL;
11075         case INSTR_JMP_EQ_HM: return NULL;
11076         case INSTR_JMP_EQ_HH: return NULL;
11077         case INSTR_JMP_EQ_I: return NULL;
11078         case INSTR_JMP_NEQ: return NULL;
11079         case INSTR_JMP_NEQ_MH: return NULL;
11080         case INSTR_JMP_NEQ_HM: return NULL;
11081         case INSTR_JMP_NEQ_HH: return NULL;
11082         case INSTR_JMP_NEQ_I: return NULL;
11083         case INSTR_JMP_LT: return NULL;
11084         case INSTR_JMP_LT_MH: return NULL;
11085         case INSTR_JMP_LT_HM: return NULL;
11086         case INSTR_JMP_LT_HH: return NULL;
11087         case INSTR_JMP_LT_MI: return NULL;
11088         case INSTR_JMP_LT_HI: return NULL;
11089         case INSTR_JMP_GT: return NULL;
11090         case INSTR_JMP_GT_MH: return NULL;
11091         case INSTR_JMP_GT_HM: return NULL;
11092         case INSTR_JMP_GT_HH: return NULL;
11093         case INSTR_JMP_GT_MI: return NULL;
11094         case INSTR_JMP_GT_HI: return NULL;
11095
11096         case INSTR_RETURN: return NULL;
11097
11098         default: return NULL;
11099         }
11100 }
11101
11102 static void
11103 action_instr_does_tx_codegen(struct action *a,
11104                         uint32_t instr_pos,
11105                         struct instruction *instr,
11106                         FILE *f)
11107 {
11108         fprintf(f,
11109                 "%s(p, t, &action_%s_instructions[%u]);\n"
11110                 "\tthread_ip_reset(p, t);\n"
11111                 "\tinstr_rx_exec(p);\n"
11112                 "\treturn;\n",
11113                 instr_type_to_func(instr),
11114                 a->name,
11115                 instr_pos);
11116 }
11117
11118 static void
11119 action_instr_extern_obj_codegen(struct action *a,
11120                                 uint32_t instr_pos,
11121                                 FILE *f)
11122 {
11123         fprintf(f,
11124                 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11125                 a->name,
11126                 instr_pos);
11127 }
11128
11129 static void
11130 action_instr_extern_func_codegen(struct action *a,
11131                                  uint32_t instr_pos,
11132                                  FILE *f)
11133 {
11134         fprintf(f,
11135                 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11136                 a->name,
11137                 instr_pos);
11138 }
11139
11140 static void
11141 action_instr_jmp_codegen(struct action *a,
11142                          uint32_t instr_pos,
11143                          struct instruction *instr,
11144                          struct instruction_data *data,
11145                          FILE *f)
11146 {
11147         switch (instr->type) {
11148         case INSTR_JMP:
11149                 fprintf(f,
11150                         "goto %s;\n",
11151                         data->jmp_label);
11152                 return;
11153
11154         case INSTR_JMP_VALID:
11155                 fprintf(f,
11156                         "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11157                         "\t\tgoto %s;\n",
11158                         a->name,
11159                         instr_pos,
11160                         data->jmp_label);
11161                 return;
11162
11163         case INSTR_JMP_INVALID:
11164                 fprintf(f,
11165                         "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11166                         "\t\tgoto %s;\n",
11167                         a->name,
11168                         instr_pos,
11169                         data->jmp_label);
11170                 return;
11171
11172         case INSTR_JMP_HIT:
11173                 fprintf(f,
11174                         "if (t->hit)\n"
11175                         "\t\tgoto %s;\n",
11176                         data->jmp_label);
11177                 return;
11178
11179         case INSTR_JMP_MISS:
11180                 fprintf(f,
11181                         "if (!t->hit)\n"
11182                         "\t\tgoto %s;\n",
11183                         data->jmp_label);
11184                 return;
11185
11186         case INSTR_JMP_ACTION_HIT:
11187                 fprintf(f,
11188                         "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11189                         "\t\tgoto %s;\n",
11190                         a->name,
11191                         instr_pos,
11192                         data->jmp_label);
11193                 return;
11194
11195         case INSTR_JMP_ACTION_MISS:
11196                 fprintf(f,
11197                         "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11198                         "\t\tgoto %s;\n",
11199                         a->name,
11200                         instr_pos,
11201                         data->jmp_label);
11202                 return;
11203
11204         case INSTR_JMP_EQ:
11205                 fprintf(f,
11206                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11207                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11208                         "\t\tgoto %s;\n",
11209                         a->name,
11210                         instr_pos,
11211                         a->name,
11212                         instr_pos,
11213                         data->jmp_label);
11214                 return;
11215
11216         case INSTR_JMP_EQ_MH:
11217                 fprintf(f,
11218                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11219                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11220                         "\t\tgoto %s;\n",
11221                         a->name,
11222                         instr_pos,
11223                         a->name,
11224                         instr_pos,
11225                         data->jmp_label);
11226                 return;
11227
11228         case INSTR_JMP_EQ_HM:
11229                 fprintf(f,
11230                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11231                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11232                         "\t\tgoto %s;\n",
11233                         a->name,
11234                         instr_pos,
11235                         a->name,
11236                         instr_pos,
11237                         data->jmp_label);
11238                 return;
11239
11240         case INSTR_JMP_EQ_HH:
11241                 fprintf(f,
11242                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11243                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11244                         "\t\tgoto %s;\n",
11245                         a->name,
11246                         instr_pos,
11247                         a->name,
11248                         instr_pos,
11249                         data->jmp_label);
11250                 return;
11251
11252         case INSTR_JMP_EQ_I:
11253                 fprintf(f,
11254                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11255                         "action_%s_instructions[%u].jmp.b_val)\n"
11256                         "\t\tgoto %s;\n",
11257                         a->name,
11258                         instr_pos,
11259                         a->name,
11260                         instr_pos,
11261                         data->jmp_label);
11262                 return;
11263
11264         case INSTR_JMP_NEQ:
11265                 fprintf(f,
11266                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11267                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11268                         "\t\tgoto %s;\n",
11269                         a->name,
11270                         instr_pos,
11271                         a->name,
11272                         instr_pos,
11273                         data->jmp_label);
11274                 return;
11275
11276         case INSTR_JMP_NEQ_MH:
11277                 fprintf(f,
11278                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11279                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11280                         "\t\tgoto %s;\n",
11281                         a->name,
11282                         instr_pos,
11283                         a->name,
11284                         instr_pos,
11285                         data->jmp_label);
11286                 return;
11287
11288         case INSTR_JMP_NEQ_HM:
11289                 fprintf(f,
11290                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11291                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11292                         "\t\tgoto %s;\n",
11293                         a->name,
11294                         instr_pos,
11295                         a->name,
11296                         instr_pos,
11297                         data->jmp_label);
11298                 return;
11299
11300         case INSTR_JMP_NEQ_HH:
11301                 fprintf(f,
11302                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11303                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11304                         "\t\tgoto %s;\n",
11305                         a->name,
11306                         instr_pos,
11307                         a->name,
11308                         instr_pos,
11309                         data->jmp_label);
11310                 return;
11311
11312         case INSTR_JMP_NEQ_I:
11313                 fprintf(f,
11314                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11315                         "action_%s_instructions[%u].jmp.b_val)\n"
11316                         "\t\tgoto %s;\n",
11317                         a->name,
11318                         instr_pos,
11319                         a->name,
11320                         instr_pos,
11321                         data->jmp_label);
11322                 return;
11323
11324         case INSTR_JMP_LT:
11325                 fprintf(f,
11326                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11327                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11328                         "\t\tgoto %s;\n",
11329                         a->name,
11330                         instr_pos,
11331                         a->name,
11332                         instr_pos,
11333                         data->jmp_label);
11334                 return;
11335
11336         case INSTR_JMP_LT_MH:
11337                 fprintf(f,
11338                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11339                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11340                         "\t\tgoto %s;\n",
11341                         a->name,
11342                         instr_pos,
11343                         a->name,
11344                         instr_pos,
11345                         data->jmp_label);
11346                 return;
11347
11348         case INSTR_JMP_LT_HM:
11349                 fprintf(f,
11350                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11351                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11352                         "\t\tgoto %s;\n",
11353                         a->name,
11354                         instr_pos,
11355                         a->name,
11356                         instr_pos,
11357                         data->jmp_label);
11358                 return;
11359
11360         case INSTR_JMP_LT_HH:
11361                 fprintf(f,
11362                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11363                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11364                         "\t\tgoto %s;\n",
11365                         a->name,
11366                         instr_pos,
11367                         a->name,
11368                         instr_pos,
11369                         data->jmp_label);
11370                 return;
11371
11372         case INSTR_JMP_LT_MI:
11373                 fprintf(f,
11374                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11375                         "action_%s_instructions[%u].jmp.b_val)\n"
11376                         "\t\tgoto %s;\n",
11377                         a->name,
11378                         instr_pos,
11379                         a->name,
11380                         instr_pos,
11381                         data->jmp_label);
11382                 return;
11383
11384         case INSTR_JMP_LT_HI:
11385                 fprintf(f,
11386                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11387                         "action_%s_instructions[%u].jmp.b_val)\n"
11388                         "\t\tgoto %s;\n",
11389                         a->name,
11390                         instr_pos,
11391                         a->name,
11392                         instr_pos,
11393                         data->jmp_label);
11394                 return;
11395
11396         case INSTR_JMP_GT:
11397                 fprintf(f,
11398                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11399                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11400                         "\t\tgoto %s;\n",
11401                         a->name,
11402                         instr_pos,
11403                         a->name,
11404                         instr_pos,
11405                         data->jmp_label);
11406                 return;
11407
11408         case INSTR_JMP_GT_MH:
11409                 fprintf(f,
11410                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11411                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11412                         "\t\tgoto %s;\n",
11413                         a->name,
11414                         instr_pos,
11415                         a->name,
11416                         instr_pos,
11417                         data->jmp_label);
11418                 return;
11419
11420         case INSTR_JMP_GT_HM:
11421                 fprintf(f,
11422                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11423                         "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11424                         "\t\tgoto %s;\n",
11425                         a->name,
11426                         instr_pos,
11427                         a->name,
11428                         instr_pos,
11429                         data->jmp_label);
11430                 return;
11431
11432         case INSTR_JMP_GT_HH:
11433                 fprintf(f,
11434                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11435                         "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11436                         "\t\tgoto %s;\n",
11437                         a->name,
11438                         instr_pos,
11439                         a->name,
11440                         instr_pos,
11441                         data->jmp_label);
11442                 return;
11443
11444         case INSTR_JMP_GT_MI:
11445                 fprintf(f,
11446                         "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11447                         "action_%s_instructions[%u].jmp.b_val)\n"
11448                         "\t\tgoto %s;\n",
11449                         a->name,
11450                         instr_pos,
11451                         a->name,
11452                         instr_pos,
11453                         data->jmp_label);
11454                 return;
11455
11456         case INSTR_JMP_GT_HI:
11457                 fprintf(f,
11458                         "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11459                         "action_%s_instructions[%u].jmp.b_val)\n"
11460                         "\t\tgoto %s;\n",
11461                         a->name,
11462                         instr_pos,
11463                         a->name,
11464                         instr_pos,
11465                         data->jmp_label);
11466                 return;
11467
11468         default:
11469                 return;
11470         }
11471 }
11472
11473 static void
11474 action_instr_return_codegen(FILE *f)
11475 {
11476         fprintf(f,
11477                 "return;\n");
11478 }
11479
11480 static void
11481 action_instr_codegen(struct action *a, FILE *f)
11482 {
11483         uint32_t i;
11484
11485         fprintf(f,
11486                 "void\n"
11487                 "action_%s_run(struct rte_swx_pipeline *p)\n"
11488                 "{\n"
11489                 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11490                 "\n",
11491                 a->name);
11492
11493         for (i = 0; i < a->n_instructions; i++) {
11494                 struct instruction *instr = &a->instructions[i];
11495                 struct instruction_data *data = &a->instruction_data[i];
11496
11497                 /* Label, if present. */
11498                 if (data->label[0])
11499                         fprintf(f, "\n%s : ", data->label);
11500                 else
11501                         fprintf(f, "\n\t");
11502
11503                 /* TX instruction type. */
11504                 if (instruction_does_tx(instr)) {
11505                         action_instr_does_tx_codegen(a, i, instr, f);
11506                         continue;
11507                 }
11508
11509                 /* Extern object/function instruction type. */
11510                 if (instr->type == INSTR_EXTERN_OBJ) {
11511                         action_instr_extern_obj_codegen(a, i, f);
11512                         continue;
11513                 }
11514
11515                 if (instr->type == INSTR_EXTERN_FUNC) {
11516                         action_instr_extern_func_codegen(a, i, f);
11517                         continue;
11518                 }
11519
11520                 /* Jump instruction type. */
11521                 if (instruction_is_jmp(instr)) {
11522                         action_instr_jmp_codegen(a, i, instr, data, f);
11523                         continue;
11524                 }
11525
11526                 /* Return instruction type. */
11527                 if (instr->type == INSTR_RETURN) {
11528                         action_instr_return_codegen(f);
11529                         continue;
11530                 }
11531
11532                 /* Any other instruction type. */
11533                 fprintf(f,
11534                         "%s(p, t, &action_%s_instructions[%u]);\n",
11535                         instr_type_to_func(instr),
11536                         a->name,
11537                         i);
11538         }
11539
11540         fprintf(f, "}\n\n");
11541 }
11542
11543 static int
11544 pipeline_codegen(struct rte_swx_pipeline *p)
11545 {
11546         struct action *a;
11547         FILE *f = NULL;
11548
11549         if (!p)
11550                 return -EINVAL;
11551
11552         /* Create the .c file. */
11553         f = fopen("/tmp/pipeline.c", "w");
11554         if (!f)
11555                 return -EIO;
11556
11557         /* Include the .h file. */
11558         fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
11559
11560         /* Add the code for each action. */
11561         TAILQ_FOREACH(a, &p->actions, node) {
11562                 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
11563
11564                 action_data_codegen(a, f);
11565
11566                 fprintf(f, "\n");
11567
11568                 action_instr_codegen(a, f);
11569
11570                 fprintf(f, "\n");
11571         }
11572
11573         /* Close the .c file. */
11574         fclose(f);
11575
11576         return 0;
11577 }
11578
11579 static int
11580 pipeline_compile(struct rte_swx_pipeline *p)
11581 {
11582         int status = 0;
11583
11584         /* Code generation. */
11585         status = pipeline_codegen(p);
11586         if (status)
11587                 return status;
11588
11589         return status;
11590 }